:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(error-tracking): judge a sample of auto-merge decisions inline (#117087)
James Gill2b1551e3e844
diff --git a/products/error_tracking/backend/temporal/fingerprint_embedding_result/activities.py b/products/error_tracking/backend/temporal/fingerprint_embedding_result/activities.pyindex 739892089357..026e138a2188 100644--- a/products/error_tracking/backend/temporal/fingerprint_embedding_result/activities.py+++ b/products/error_tracking/backend/temporal/fingerprint_embedding_result/activities.py@@ -1,10 +1,8 @@import mathimport timeimport uuid-from datetime import datetime, timedeltafrom django.conf import settings-from django.utils.dateparse import parse_datetimefrom posthog.hogql import astfrom posthog.hogql.constants import HogQLGlobalSettings@@ -36,12 +34,15 @@ from products.error_tracking.backend.temporal.fingerprint_embedding_result.judgeload_auto_merge_config,)from products.error_tracking.backend.temporal.fingerprint_embedding_result.types import (+ CLOSEST_FINGERPRINTS_WINDOW,AutoMergeOutcome,AutoMergeReopenedTarget,FingerprintEmbeddingMergeResult,FingerprintEmbeddingResultInputs,SimilarFingerprintDistance,+ parse_embedding_timestamp,)+from products.error_tracking.backend.temporal.grouping_judge.inline import grouping_judge_propertiesfrom products.error_tracking.backend.temporal.lifecycle.types import LifecycleIssueSnapshotNIL_UUID = "00000000-0000-0000-0000-000000000000"@@ -147,19 +148,12 @@ def _model_specific_query(query_by_model: dict[str, str], model_name: str) -> streturn query-def _parse_timestamp(timestamp: str) -> datetime:- parsed = parse_datetime(timestamp)- if parsed is None:- raise ValueError(f"Invalid embedding timestamp: {timestamp}")- return parsed--def _closest_fingerprints_query(inputs: FingerprintEmbeddingResultInputs,model_name: str,target_embedding: list[float],) -> ast.SelectQuery | ast.SelectSetQuery:- min_timestamp = _parse_timestamp(inputs.timestamp) - timedelta(days=30)+ min_timestamp = parse_embedding_timestamp(inputs.timestamp) - CLOSEST_FINGERPRINTS_WINDOWreturn parse_select(_model_specific_query(CLOSEST_FINGERPRINTS_QUERY_BY_MODEL, model_name),placeholders={@@ -212,6 +206,7 @@ def _report_closest_fingerprint_metrics(closest_fingerprints: list[SimilarFingerprintDistance],model_name: str,query_duration_ms: float,+ judge_properties: dict[str, object],) -> None:properties: dict[str, object] = {"model_name": model_name,@@ -219,6 +214,7 @@ def _report_closest_fingerprint_metrics("fingerprint": inputs.fingerprint,"query_duration_ms": query_duration_ms,"closest_fingerprint_count": len(closest_fingerprints),+ **judge_properties,}for index, closest_fingerprint in enumerate(closest_fingerprints[:3], start=1):properties[f"rank_{index}_fingerprint"] = closest_fingerprint.fingerprint@@ -309,7 +305,12 @@ def _merge_into_candidate(reopened_target = (_reopened_target(target_issue, event_reference=event_reference) if merge_outcome.reopened else None)- return AutoMergeOutcome(merged_count=1, reopened_target=reopened_target)+ return AutoMergeOutcome(+ merged_count=1,+ reopened_target=reopened_target,+ target_fingerprint=candidate.fingerprint,+ merge_decision=str(decision_properties["merge_decision"]),+ )def _fetch_fingerprint_document(@@ -322,7 +323,9 @@ def _fetch_fingerprint_document("rendering": ast.Constant(value=inputs.rendering),"team_id": ast.Constant(value=team.id),"fingerprint": ast.Constant(value=fingerprint),- "min_timestamp": ast.Constant(value=_parse_timestamp(inputs.timestamp) - timedelta(days=30)),+ "min_timestamp": ast.Constant(+ value=parse_embedding_timestamp(inputs.timestamp) - CLOSEST_FINGERPRINTS_WINDOW+ ),},)response = execute_hogql_query(@@ -505,20 +508,30 @@ def merge_similar_fingerprints(query_duration_seconds = time.monotonic() - startquery_duration_ms = query_duration_seconds * 1000- # Keep emitting candidate metrics for every run; merging is gated separately by configuration.- _report_closest_fingerprint_metrics(team, inputs, closest_fingerprints, inputs.model_name, query_duration_ms)- merge_outcome = _merge_fingerprint_into_closest_issue(- team,- inputs.fingerprint,- closest_fingerprints,- inputs.source_issue_id,- event_reference=_event_reference(inputs),- # Only read the flag when there is something to decide.- config=load_auto_merge_config(team)+ # Only read the flag when there is something to decide.+ config = (+ load_auto_merge_config(team)if settings.ERROR_TRACKING_AUTO_MERGE_ENABLED and closest_fingerprints- else DEFAULT_AUTO_MERGE_CONFIG,- judge_inputs=inputs,+ else DEFAULT_AUTO_MERGE_CONFIG)+ merge_outcome: AutoMergeOutcome | None = None+ try:+ merge_outcome = _merge_fingerprint_into_closest_issue(+ team,+ inputs.fingerprint,+ closest_fingerprints,+ inputs.source_issue_id,+ event_reference=_event_reference(inputs),+ config=config,+ judge_inputs=inputs,+ )+ finally:+ # Keep emitting candidate metrics for every run; merging is gated separately by configuration.+ # The grouping judge runs only after the merge decision, so it cannot change it.+ judge_properties = grouping_judge_properties(team, inputs, closest_fingerprints, merge_outcome, config)+ _report_closest_fingerprint_metrics(+ team, inputs, closest_fingerprints, inputs.model_name, query_duration_ms, judge_properties+ )return FingerprintEmbeddingMergeResult(merged_count=merge_outcome.merged_count,diff --git a/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_fingerprint_embedding_result.py b/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_fingerprint_embedding_result.pyindex b9a37e9a7bf1..d88fb1f2a687 100644--- a/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_fingerprint_embedding_result.py+++ b/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_fingerprint_embedding_result.py@@ -1,6 +1,8 @@import uuid+import threadingfrom datetime import UTC, datetime-from typing import cast+from types import SimpleNamespace+from typing import Any, castimport pytestfrom posthog.test.base import BaseTest@@ -9,6 +11,9 @@ from unittest.mock import MagicMock, patchfrom django.db import OperationalErrorfrom django.test import override_settings+import httpx+from anthropic import APIConnectionError+from parameterized import parameterizedfrom temporalio.exceptions import CancelledErrorfrom posthog.clickhouse.client.connection import ClickHouseUser, Workload@@ -38,10 +43,13 @@ from products.error_tracking.backend.temporal.fingerprint_embedding_result.typesFingerprintEmbeddingResultInputs,SimilarFingerprintDistance,)+from products.error_tracking.backend.temporal.grouping_judge.config import JudgeConfigfrom products.error_tracking.backend.temporal.lifecycle.issue_created.activities import (merge_issue_created_fingerprint_activity,)+JUDGE_MODULE = "products.error_tracking.backend.temporal.grouping_judge.inline"+def _inputs() -> FingerprintEmbeddingResultInputs:return FingerprintEmbeddingResultInputs(@@ -139,6 +147,7 @@ class TestFingerprintEmbeddingResultActivity:closest_fingerprints=closest_fingerprints,model_name="text-embedding-3-large-3072",query_duration_ms=12.3,+ judge_properties={"live_decision": "merge"},)properties = capture.call_args.kwargs["properties"]@@ -148,6 +157,7 @@ class TestFingerprintEmbeddingResultActivity:assert properties["rank_1_distance"] == 0.01assert properties["rank_2_fingerprint"] == "fingerprint-2"assert properties["rank_3_fingerprint"] == "fingerprint-3"+ assert properties["live_decision"] == "merge"def test_merge_activity_reports_distances(self) -> None:closest_fingerprints = [@@ -218,6 +228,44 @@ class TestFingerprintEmbeddingResultActivity:assert capture_activity_exception.called is expect_captured+ def test_merge_activity_reports_an_error_decision_and_reraises_when_the_merge_raises(self) -> None:+ error = StaleAutoMergeStateError("ownership changed")+ capture = MagicMock()++ with (+ patch(+ "products.error_tracking.backend.temporal.fingerprint_embedding_result.activities.Team.objects.get",+ return_value=MagicMock(id=1, uuid=uuid.uuid4()),+ ),+ patch(+ "products.error_tracking.backend.temporal.fingerprint_embedding_result.activities._query_closest_fingerprints",+ return_value=[SimilarFingerprintDistance(fingerprint="fingerprint-1", distance=0.01)],+ ),+ patch(+ "products.error_tracking.backend.temporal.fingerprint_embedding_result.activities._merge_fingerprint_into_closest_issue",+ side_effect=error,+ ),+ patch(+ "products.error_tracking.backend.temporal.fingerprint_embedding_result.activities.ph_background_capture",+ return_value=capture,+ ),+ patch(+ "products.error_tracking.backend.temporal.fingerprint_embedding_result.activities.groups",+ return_value={},+ ),+ patch(+ "products.error_tracking.backend.temporal.fingerprint_embedding_result.activities._capture_activity_exception"+ ),+ pytest.raises(StaleAutoMergeStateError) as raised,+ ):+ merge_issue_created_fingerprint_activity(_inputs())++ assert raised.value is error+ (call,) = capture.call_args_list+ assert call.kwargs["event"] == "error_tracking_fingerprint_embedding_result_metrics"+ assert call.kwargs["properties"]["live_decision"] == "error"+ assert call.kwargs["properties"]["live_target_fingerprint"] is None+def test_merge_fingerprint_skips_when_auto_merge_disabled(self) -> None:with override_settings(ERROR_TRACKING_AUTO_MERGE_ENABLED=False):result = _merge_fingerprint_into_closest_issue(@@ -566,3 +614,105 @@ class TestAutoMergeReopensTarget(BaseTest):assert outcome.merged_count == 1assert outcome.reopened_target is not Noneassert outcome.reopened_target.issue_id == str(target.id)+++@override_settings(ERROR_TRACKING_AUTO_MERGE_ENABLED=True)+class TestGroupingJudgeNeverChangesTheMerge(BaseTest):+ def setUp(self) -> None:+ super().setUp()+ self.organization.is_ai_data_processing_approved = True+ self.organization.save()+ for fingerprint in ("fp-source", "fp-target"):+ issue = ErrorTrackingIssue.objects.create(team=self.team, name="TypeError")+ ErrorTrackingIssueFingerprintV2.objects.create(team=self.team, issue=issue, fingerprint=fingerprint)+ self.target_issue_id = ErrorTrackingIssueFingerprintV2.objects.get(fingerprint="fp-target").issue_id++ def _run(self, client: MagicMock | None = None, build_error: Exception | None = None) -> tuple[Any, list[Any]]:+ inputs = FingerprintEmbeddingResultInputs(+ team_id=self.team.id,+ fingerprint="fp-source",+ rendering="type_message_and_stack",+ timestamp="2026-06-08T00:00:00Z",+ model_name="text-embedding-3-large-3072",+ embedding=[0.1],+ )+ texts = MagicMock(results=[("fp-source", "TypeError: a"), ("fp-target", "TypeError: b")])+ capture = MagicMock()+ with (+ patch(+ "products.error_tracking.backend.temporal.fingerprint_embedding_result.activities._query_closest_fingerprints",+ return_value=[SimilarFingerprintDistance(fingerprint="fp-target", distance=0.01)],+ ),+ patch(+ "products.error_tracking.backend.temporal.fingerprint_embedding_result.activities.ph_background_capture",+ return_value=capture,+ ),+ patch(f"{JUDGE_MODULE}.execute_hogql_query", return_value=texts),+ patch(f"{JUDGE_MODULE}.build_ai_gateway_anthropic_client", return_value=client, side_effect=build_error),+ patch(f"{JUDGE_MODULE}.TOTAL_BUDGET_SECONDS", 0.5),+ patch(f"{JUDGE_MODULE}.current_judge_config", return_value=JudgeConfig(sample_rate=1.0)),+ ):+ result = merge_issue_created_fingerprint_activity(inputs)+ events = [+ call.kwargs["properties"]+ for call in capture.call_args_list+ if call.kwargs["event"] == "error_tracking_fingerprint_embedding_result_metrics"+ ]+ return result, events++ @parameterized.expand(+ [+ ("judged", "judged"),+ ("judge_raises", "unavailable"),+ ("gateway_not_configured", "unavailable"),+ ("gateway_down", "unavailable"),+ ("judge_hangs", "timeout"),+ ("worker_drains_during_the_judge", "cancelled"),+ ]+ )+ def test_merge_and_event_survive_any_judge_failure(self, case: str, outcome: str) -> None:+ release = threading.Event()+ client = MagicMock()+ create = client.with_options.return_value.messages.create+ build_error: Exception | None = None+ if case == "judged":+ create.return_value = SimpleNamespace(+ content=[SimpleNamespace(type="tool_use", name="record_verdict", input={"same_issue": True})]+ )+ elif case == "judge_raises":+ build_error = RuntimeError("judge bug")+ elif case == "gateway_not_configured":+ build_error = ValueError("AI_GATEWAY_URL and AI_GATEWAY_API_KEY must be configured")+ elif case == "gateway_down":+ create.side_effect = APIConnectionError(request=httpx.Request("POST", "http://gateway.example.com"))+ elif case == "worker_drains_during_the_judge":+ create.side_effect = CancelledError("Cancelled")+ else:+ create.side_effect = lambda **_: release.wait(timeout=5)++ try:+ result, events = self._run(client, build_error)+ finally:+ release.set()++ assert (result.merged_count, result.reopened_target, result.closest_fingerprints) == (+ 1,+ None,+ [SimilarFingerprintDistance(fingerprint="fp-target", distance=0.01)],+ )+ assert ErrorTrackingIssueFingerprintV2.objects.get(fingerprint="fp-source").issue_id == self.target_issue_id+ (event,) = events+ assert (+ event["live_decision"],+ event["live_target_fingerprint"],+ event["live_merge_decision"],+ event["judge_outcome"],+ ) == ("merge", "fp-target", "distance", outcome)++ def test_merge_and_event_survive_a_failure_outside_the_judge_call(self) -> None:+ with patch(f"{JUDGE_MODULE}.band_for", side_effect=RuntimeError("judge bug")):+ result, events = self._run()++ assert result.merged_count == 1+ assert ErrorTrackingIssueFingerprintV2.objects.get(fingerprint="fp-source").issue_id == self.target_issue_id+ assert [event["live_decision"] for event in events] == ["merge"]diff --git a/products/error_tracking/backend/temporal/fingerprint_embedding_result/types.py b/products/error_tracking/backend/temporal/fingerprint_embedding_result/types.pyindex 9c32b7968285..9e01bda7b4dd 100644--- a/products/error_tracking/backend/temporal/fingerprint_embedding_result/types.py+++ b/products/error_tracking/backend/temporal/fingerprint_embedding_result/types.py@@ -1,7 +1,19 @@import dataclasses+from datetime import datetime, timedelta++from django.utils.dateparse import parse_datetimefrom products.error_tracking.backend.temporal.lifecycle.types import LifecycleIssueSnapshot+CLOSEST_FINGERPRINTS_WINDOW = timedelta(days=30)+++def parse_embedding_timestamp(timestamp: str) -> datetime:+ parsed = parse_datetime(timestamp)+ if parsed is None:+ raise ValueError(f"Invalid embedding timestamp: {timestamp}")+ return parsed+@dataclasses.dataclass(frozen=True)class FingerprintEmbeddingResultInputs:@@ -38,6 +50,9 @@ class AutoMergeReopenedTarget:class AutoMergeOutcome:merged_count: int = 0reopened_target: AutoMergeReopenedTarget | None = None+ target_fingerprint: str | None = None+ # Which rule made the merge: "distance" or "judge". None when nothing merged in this attempt.+ merge_decision: str | None = None@dataclasses.dataclass(frozen=True)diff --git a/products/error_tracking/backend/temporal/grouping_judge/algorithms.py b/products/error_tracking/backend/temporal/grouping_judge/algorithms.pynew file mode 100644index 000000000000..73a46a12c4dc--- /dev/null+++ b/products/error_tracking/backend/temporal/grouping_judge/algorithms.py@@ -0,0 +1,29 @@+from collections.abc import Callable++from posthog.dataclasses import frozen++from products.error_tracking.backend.temporal.grouping_judge.types import Decision+++@frozen+class GroupingAlgorithm:+ id: str+ decide: Callable[[float], Decision]+++# A fixed copy of the live threshold. A change to the live threshold is a new algorithm, so rows+# recorded under this id stay comparable.+EMBEDDING_THRESHOLD_V1_DISTANCE = 0.019++EMBEDDING_THRESHOLD_V1 = GroupingAlgorithm(+ id="embedding_threshold_v1",+ decide=lambda distance: "merge" if distance < EMBEDDING_THRESHOLD_V1_DISTANCE else "no_merge",+)++ALGORITHMS: tuple[GroupingAlgorithm, ...] = (EMBEDDING_THRESHOLD_V1,)+# The algorithm production runs. Point this at another entry when production switches algorithm.+LIVE_ALGORITHM = EMBEDDING_THRESHOLD_V1+++def decisions_for(distance: float, algorithms: tuple[GroupingAlgorithm, ...] = ALGORITHMS) -> dict[str, Decision]:+ return {algorithm.id: algorithm.decide(distance) for algorithm in algorithms}diff --git a/products/error_tracking/backend/temporal/grouping_judge/config.py b/products/error_tracking/backend/temporal/grouping_judge/config.pynew file mode 100644index 000000000000..b59268cee11d--- /dev/null+++ b/products/error_tracking/backend/temporal/grouping_judge/config.py@@ -0,0 +1,71 @@+import json++import structlog+import posthoganalytics++from posthog.dataclasses import frozen+from posthog.utils import get_instance_region++logger = structlog.get_logger(__name__)++REMOTE_CONFIG_KEY = "error-tracking-grouping-judge"+REMOTE_CONFIG_DISTINCT_ID = "error_tracking_grouping_judge"+++@frozen+class JudgeConfig:+ sample_rate: float = 0.0+ # Empty means every team.+ team_ids: frozenset[int] = frozenset()++ def is_enabled_for(self, team_id: int) -> bool:+ return self.sample_rate > 0 and (not self.team_ids or team_id in self.team_ids)+++JUDGE_OFF = JudgeConfig()+++def parse_judge_config(payload: object, region: str | None) -> JudgeConfig:+ """Read this region's entry, e.g. `{"EU": {"sample_rate": 0.01, "team_ids": []}}`.++ Team ids repeat across regions, so each region has its own entry. Anything missing or malformed is off.+ """+ if isinstance(payload, str):+ try:+ payload = json.loads(payload)+ except ValueError:+ return JUDGE_OFF+ if not isinstance(payload, dict) or region is None:+ return JUDGE_OFF+ entry = payload.get(region)+ if not isinstance(entry, dict):+ return JUDGE_OFF+ sample_rate = entry.get("sample_rate", 0)+ team_ids = entry.get("team_ids", [])+ if (+ isinstance(sample_rate, bool)+ or not isinstance(sample_rate, int | float)+ or not 0 <= sample_rate <= 1+ or not isinstance(team_ids, list)+ or not all(isinstance(team_id, int) and not isinstance(team_id, bool) for team_id in team_ids)+ ):+ logger.warning("error_tracking.grouping_judge.invalid_remote_config", region=region)+ return JUDGE_OFF+ return JudgeConfig(sample_rate=float(sample_rate), team_ids=frozenset(team_ids))+++def current_judge_config() -> JudgeConfig:+ # Reads the payload from the locally evaluated flag definitions, so the merge activity makes no network call.+ # A missing or disabled flag returns None, which turns the judge off.+ try:+ # With no definitions loaded, the SDK loads them with a blocking network call even for a+ # local-only read, so the judge stays off until the background poller loads them.+ if posthoganalytics.feature_flag_definitions() is None:+ return JUDGE_OFF+ payload = posthoganalytics.get_feature_flag_payload(+ REMOTE_CONFIG_KEY, REMOTE_CONFIG_DISTINCT_ID, only_evaluate_locally=True, send_feature_flag_events=False+ )+ except Exception:+ logger.exception("error_tracking.grouping_judge.remote_config_failed")+ return JUDGE_OFF+ return parse_judge_config(payload, get_instance_region())diff --git a/products/error_tracking/backend/temporal/grouping_judge/inline.py b/products/error_tracking/backend/temporal/grouping_judge/inline.pynew file mode 100644index 000000000000..6a8b66418a4f--- /dev/null+++ b/products/error_tracking/backend/temporal/grouping_judge/inline.py@@ -0,0 +1,286 @@+"""Judge a sample of auto-merge decisions for the metrics event. Nothing here can change a merge."""++import time+import threading+from concurrent.futures import (+ ThreadPoolExecutor,+ TimeoutError as FuturesTimeoutError,+)++import structlog+import temporalio.exceptions+from anthropic import Anthropic++from posthog.hogql import ast+from posthog.hogql.constants import HogQLGlobalSettings+from posthog.hogql.parser import parse_select+from posthog.hogql.query import execute_hogql_query++from posthog.clickhouse.client.connection import ClickHouseUser, Workload+from posthog.dataclasses import frozen+from posthog.llm.gateway_client import build_ai_gateway_anthropic_client, team_distinct_id+from posthog.models import Team+from posthog.temporal.common.shutdown import WorkerShuttingDownError+from posthog.temporal.common.utils import close_db_connections++from products.error_tracking.backend.temporal.fingerprint_embedding_result.judge import (+ DEFAULT_AUTO_MERGE_CONFIG,+ AutoMergeConfig,+ _truncate,+)+from products.error_tracking.backend.temporal.fingerprint_embedding_result.types import (+ CLOSEST_FINGERPRINTS_WINDOW,+ AutoMergeOutcome,+ FingerprintEmbeddingResultInputs,+ SimilarFingerprintDistance,+ parse_embedding_timestamp,+)+from products.error_tracking.backend.temporal.grouping_judge.algorithms import LIVE_ALGORITHM, decisions_for+from products.error_tracking.backend.temporal.grouping_judge.config import current_judge_config+from products.error_tracking.backend.temporal.grouping_judge.judge import JUDGE_MODEL, JudgeVerdict, judge_pair+from products.error_tracking.backend.temporal.grouping_judge.sampling import (+ JudgeSample,+ band_for,+ distance_bucket_for,+ sample_pair,+)+from products.error_tracking.backend.temporal.grouping_judge.types import JudgeOutcome, LiveDecision++logger = structlog.get_logger(__name__)++AI_PRODUCT = "error_tracking_grouping_judge"+# Bounds the ClickHouse text fetch and every judge call. A sampled issue's alert waits at most this long for them.+TOTAL_BUDGET_SECONDS = 10.0+TEXT_QUERY_MAX_EXECUTION_SECONDS = 3+# A re-judge with less time than this left would most likely time out, so it is skipped.+MIN_REJUDGE_SECONDS = 2.0++_DOCUMENT_TEXT_QUERY_TEMPLATE = """+ SELECT document_id, argMax(content, inserted_at)+ FROM {table}+ WHERE product = 'error_tracking'+ AND document_type = 'fingerprint'+ AND rendering = {{rendering}}+ AND team_id = {{team_id}}+ AND timestamp >= {{min_timestamp}}+ AND document_id IN ({{document_a}}, {{document_b}})+ GROUP BY document_id+ """+# Reads the same model-specific table as the closest-fingerprints query, so both texts are in it.+DOCUMENT_TEXT_QUERY_BY_MODEL = {+ model_name: _DOCUMENT_TEXT_QUERY_TEMPLATE.format(table=table)+ for model_name, table in (+ ("text-embedding-3-large-3072", "document_embeddings_text_embedding_3_large_3072"),+ ("text-embedding-3-small-1536", "document_embeddings_text_embedding_3_small_1536"),+ )+}++# The judge runs on its own thread so the activity can stop waiting at the budget even when a+# call ignores its own timeout. A timed-out call keeps its thread until its own timeouts end it.+# The ClickHouse socket has no client timeout, so a sample that finds every thread busy skips the+# judge instead of queueing behind threads that may never come back.+_MAX_WORKERS = 4+_executor = ThreadPoolExecutor(max_workers=_MAX_WORKERS, thread_name_prefix="error-tracking-grouping-judge")+_free_workers = threading.BoundedSemaphore(_MAX_WORKERS)+++@frozen+class _JudgeResult:+ outcome: JudgeOutcome+ same_issue: bool | None = None+ rejudge_same_issue: bool | None = None+++def _is_cancellation(error: BaseException) -> bool:+ return temporalio.exceptions.is_cancelled_exception(error) or isinstance(error, WorkerShuttingDownError)+++def _live_decision(merge_outcome: AutoMergeOutcome | None) -> LiveDecision:+ if merge_outcome is None:+ return "error"+ return "merge" if merge_outcome.merged_count > 0 else "no_merge"+++def _fetch_document_texts(+ team: Team, inputs: FingerprintEmbeddingResultInputs, neighbour_fingerprint: str+) -> dict[str, str]:+ query_text = DOCUMENT_TEXT_QUERY_BY_MODEL.get(inputs.model_name)+ if query_text is None:+ raise ValueError(f"No document text query for embedding model: {inputs.model_name}")+ # The closest-fingerprints window, so the neighbour's row is inside it.+ min_timestamp = parse_embedding_timestamp(inputs.timestamp) - CLOSEST_FINGERPRINTS_WINDOW+ query = parse_select(+ query_text,+ placeholders={+ "rendering": ast.Constant(value=inputs.rendering),+ "team_id": ast.Constant(value=team.id),+ "min_timestamp": ast.Constant(value=min_timestamp),+ "document_a": ast.Constant(value=inputs.fingerprint),+ "document_b": ast.Constant(value=neighbour_fingerprint),+ },+ )+ response = execute_hogql_query(+ query=query,+ team=team,+ query_type="ErrorTrackingGroupingJudgeDocumentTexts",+ workload=Workload.OFFLINE,+ ch_user=ClickHouseUser.ERROR_TRACKING,+ settings=HogQLGlobalSettings(max_execution_time=TEXT_QUERY_MAX_EXECUTION_SECONDS),+ )+ return {row[0]: row[1] for row in response.results if row[1]}+++def _judge_once(+ client: Anthropic, *, new_text: str, neighbour_text: str, swapped: bool, deadline: float+) -> JudgeVerdict:+ remaining = deadline - time.monotonic()+ if remaining <= 0:+ return JudgeVerdict(outcome="timeout")+ first, second = (neighbour_text, new_text) if swapped else (new_text, neighbour_text)+ return judge_pair(client, document_a=first, document_b=second, timeout_seconds=remaining)+++@close_db_connections+def _judge_in_worker(+ team: Team,+ inputs: FingerprintEmbeddingResultInputs,+ neighbour_fingerprint: str,+ sample: JudgeSample,+ deadline: float,+) -> _JudgeResult:+ # A task that waited in the queue past the deadline has no caller left to read its result.+ if time.monotonic() >= deadline:+ return _JudgeResult(outcome="timeout")+ texts = _fetch_document_texts(team, inputs, neighbour_fingerprint)+ # A workflow started before the activity input carried the text has to read it from ClickHouse.+ new_text = inputs.content or texts.get(inputs.fingerprint)+ neighbour_text = texts.get(neighbour_fingerprint)+ if new_text is None or neighbour_text is None:+ return _JudgeResult(outcome="no_text")+ # Keeps the message and the throw site, which the renderer puts at the end of a long stack.+ new_text, neighbour_text = _truncate(new_text), _truncate(neighbour_text)++ try:+ client = build_ai_gateway_anthropic_client(+ ai_product=AI_PRODUCT, team_id=team.id, distinct_id=team_distinct_id(team.id)+ )+ except ValueError:+ return _JudgeResult(outcome="unavailable")+ with client:+ verdict = _judge_once(+ client, new_text=new_text, neighbour_text=neighbour_text, swapped=sample.swapped, deadline=deadline+ )+ if verdict.outcome != "judged" or not sample.rejudge or deadline - time.monotonic() < MIN_REJUDGE_SECONDS:+ return _JudgeResult(outcome=verdict.outcome, same_issue=verdict.same_issue)+ rejudge = _judge_once(+ client, new_text=new_text, neighbour_text=neighbour_text, swapped=not sample.swapped, deadline=deadline+ )+ return _JudgeResult(outcome=verdict.outcome, same_issue=verdict.same_issue, rejudge_same_issue=rejudge.same_issue)+++def _judge(+ team: Team, inputs: FingerprintEmbeddingResultInputs, neighbour_fingerprint: str, sample: JudgeSample+) -> _JudgeResult:+ deadline = time.monotonic() + TOTAL_BUDGET_SECONDS+ try:+ # A null approval counts as not approved.+ if team.organization.is_ai_data_processing_approved is not True:+ return _JudgeResult(outcome="no_consent")+ if not _free_workers.acquire(blocking=False):+ return _JudgeResult(outcome="unavailable")+ try:+ future = _executor.submit(_judge_in_worker, team, inputs, neighbour_fingerprint, sample, deadline)+ except BaseException:+ _free_workers.release()+ raise+ future.add_done_callback(lambda _: _free_workers.release())+ return future.result(timeout=max(deadline - time.monotonic(), 0))+ except FuturesTimeoutError:+ return _JudgeResult(outcome="timeout")+ except Exception as error:+ if _is_cancellation(error):+ # A worker drain raises into this thread while it waits. The merge has committed, and a+ # retry cannot recover its reopened target, so the activity must still return its result.+ return _JudgeResult(outcome="cancelled")+ # Caught here rather than only at the entry point, so the row keeps its other judge properties.+ logger.exception("error_tracking.grouping_judge.failed", team_id=team.id)+ return _JudgeResult(outcome="unavailable")+++def _grouping_judge_properties(+ team: Team,+ inputs: FingerprintEmbeddingResultInputs,+ closest_fingerprints: list[SimilarFingerprintDistance],+ merge_outcome: AutoMergeOutcome | None,+ merge_config: AutoMergeConfig,+) -> dict[str, object]:+ properties: dict[str, object] = {+ "live_decision": _live_decision(merge_outcome),+ "live_target_fingerprint": merge_outcome.target_fingerprint if merge_outcome is not None else None,+ "live_merge_decision": merge_outcome.merge_decision if merge_outcome is not None else None,+ # The fixed rule describes the merge only while the flag leaves every setting at its default.+ "live_algorithm": LIVE_ALGORITHM.id if merge_config == DEFAULT_AUTO_MERGE_CONFIG else "auto_merge_config",+ "live_auto_merge_threshold": merge_config.auto_merge_threshold,+ "live_judge_enabled": merge_config.judge_enabled,+ "live_judge_shadow": merge_config.judge_shadow,+ "live_judge_cutoff": merge_config.judge_cutoff,+ "live_judge_max_distance": merge_config.judge_max_distance,+ "decisions": {},+ "judge_band": None,+ "distance_bucket": None,+ "judge_sampled": False,+ }+ if not closest_fingerprints:+ return properties+ nearest = closest_fingerprints[0]+ band = band_for(nearest.distance)+ properties.update(+ decisions=decisions_for(nearest.distance),+ judge_band=band,+ distance_bucket=distance_bucket_for(nearest.distance),+ )+ # A failed merge is retried, and the retry judges the pair, so judging it here would pay twice.+ if merge_outcome is None:+ return properties+ config = current_judge_config()+ if not config.is_enabled_for(team.id):+ return properties+ sample = sample_pair(team.id, inputs.fingerprint, config.sample_rate)+ if not sample.sampled:+ return properties++ # Judge the pair the merge used, which is not rank 1 when the merge skipped it.+ judged_fingerprint = merge_outcome.target_fingerprint or nearest.fingerprint+ start = time.monotonic()+ result = _judge(team, inputs, judged_fingerprint, sample)+ # Document text and the model's reasoning hold customer data, so neither goes in the event.+ return {+ **properties,+ "judge_sampled": True,+ "judge_target_fingerprint": judged_fingerprint,+ "judge_outcome": result.outcome,+ "judge_same_issue": result.same_issue,+ "judge_swapped": sample.swapped,+ "judge_rejudge_same_issue": result.rejudge_same_issue,+ "judge_model": JUDGE_MODEL,+ "judge_view": inputs.rendering,+ "judge_duration_ms": (time.monotonic() - start) * 1000,+ }+++def grouping_judge_properties(+ team: Team,+ inputs: FingerprintEmbeddingResultInputs,+ closest_fingerprints: list[SimilarFingerprintDistance],+ merge_outcome: AutoMergeOutcome | None,+ merge_config: AutoMergeConfig = DEFAULT_AUTO_MERGE_CONFIG,+) -> dict[str, object]:+ """Decision and judge properties for the metrics event. `merge_outcome` is None when the merge raised.++ Never raises, and returns within about `TOTAL_BUDGET_SECONDS`.+ """+ try:+ return _grouping_judge_properties(team, inputs, closest_fingerprints, merge_outcome, merge_config)+ except Exception:+ logger.exception("error_tracking.grouping_judge.properties_failed", team_id=team.id)+ return {"live_decision": _live_decision(merge_outcome)}diff --git a/products/error_tracking/backend/temporal/grouping_judge/judge.py b/products/error_tracking/backend/temporal/grouping_judge/judge.pynew file mode 100644index 000000000000..f663f3f92d8c--- /dev/null+++ b/products/error_tracking/backend/temporal/grouping_judge/judge.py@@ -0,0 +1,156 @@+"""The LLM judge. It sees two document texts and nothing about how they were paired."""++from anthropic import Anthropic, AnthropicError, APIConnectionError, APIStatusError, APITimeoutError+from anthropic.types import ToolChoiceToolParam, ToolParam++from posthog.dataclasses import frozen++from products.error_tracking.backend.temporal.grouping_judge.types import JudgeOutcome++JUDGE_MODEL = "claude-haiku-4-5"+# Room for the reasoning, which comes before the verdict.+MAX_TOKENS = 768+# Pins the call to Bedrock in the caller's region, so EU documents never leave EU Bedrock.+PROVIDER_HEADERS = {"X-PostHog-Provider": "bedrock"}++SYSTEM_PROMPT = """\+You are an expert in error tracking and crash triage. You will be shown two error documents, each rendered \+as an error type + message followed by a stack trace. They come from the same application but were assigned \+different grouping fingerprints.++Decide whether they represent the SAME underlying issue — i.e. a developer fixing one would consider the \+other a duplicate that should be grouped with it, because one fix resolves both.++How to decide:+1. Find the THROW SITE of each document: the error type, the message, and the application frames nearest to \+where the error was raised. Frames are usually listed from the outermost caller first to the throw site last \+(numbered frames: #00 is the throw site).+2. If the throw sites match, the documents are the SAME issue even when they differ in:+- dynamic values anywhere: IDs, URLs and URL path segments, numbers, percentages, timestamps, memory \+addresses, thread/process names, session ids, user input, in any language;+- line/column numbers, build hashes in file names, bundler module ids and chunk names, and minified frame \+names (1–3 character identifiers such as a, Qx, b7 that change between builds);+- the outer callers above the throw site, extra or missing frames at the bottom of the stack, stacks \+truncated at a different depth (e.g. stack overflows), and frames inside framework or runtime internals \+(React reconciler/scheduler, webpack runtime, unsymbolicated VM addresses);+- a frame whose "file" is a page URL (/pricing vs /blog): it is the same inline, injected or single-page-app \+script running on another page; compare its function names instead. When those frames have no function \+names, they count as the same script only if the message is identical and the frame shape matches; inline \+event handlers (onclick, ...) belong to their own page and are not shared;+- the variable part of a templated message when the error type, template and throw site match: the requested \+path or asset in not-found / failed-to-load errors (404 for /a vs /b, a failed-to-load error for different \+assets), or a list of rejected fields from the same validator. When the variable part is itself an error \+message, compare that inner error as its own throw site;+- which chunk or module failed in chunk / dynamic-import load failures (ChunkLoadError, "error loading \+dynamically imported module"): one issue (deploy skew or network);+- which caller hit the same failing dependency in transient transport-level failures (no HTTP response: \+network error, aborted request, timeout, DNS or TLS failure, connection pool exhausted, or HTTP \+502/503/504): group by the failure, not by the caller. An HTTP 4xx or 500 response is never transient;+- which hook or component hit a null React dispatcher ("reading 'useState'", 'useContext', ... thrown inside \+React's hook exports): one issue (duplicate React copy or hook called outside render);+- the page on which a browser- or extension-injected script failed (window.ethereum, __firefox__, \+_AutofillCallbackHandler).+3. They are DIFFERENT issues when the throw sites differ:+- different meaningful application files, components, handlers or functions nearest the throw site (when the \+error is raised inside framework or library code, the first application frames calling into it decide) \+(CartSummary.tsx vs OrderHistory.tsx, component Checkout vs Profile), even with an identical message: the \+same failure in two places needs two fixes. Named frames without a file (App, loadSettings) are application \+frames, not minified names;+- with no application frames at all, the library frame that calls into the framework stands in for the \+application frame (a websocket library vs a state library, each calling into react-dom);+- names that come from the application's own code differ: the undefined identifier (googletag vs jQuery is \+not defined), the missing property, key or module, a nested error class in the message (NetworkError vs \+AbortError), a different HTTP status (404 vs 401, 500 vs 503) or an HTTP status vs a network error, or a \+different operation;+- React minified errors (e.g. #418) with a different error code, OR different args (text vs HTML), OR \+different route-specific files or components (settings-*.js vs billing-*.js): any one of these makes them \+different;+- the error type or root message differs in meaning, not just in dynamic values.++Examples (SAME): "Page not found: /old-page" vs "Page not found: /docs/missing" with the same stack; "Failed \+to load asset: /assets/icons/close.svg" vs "/assets/logo.svg"; the same recursion frames on /pricing vs \+/blog; a connection-pool-exhausted error reached from two different request handlers; a validation error \+listing different rejected fields from the same validator.+Examples (DIFFERENT): "googletag is not defined" vs "jQuery is not defined"; "status code 404" vs "Network \+Error"; the same TypeError thrown in component Checkout vs component Profile.++Before deciding, name the throw site of each document. Then check the DIFFERENT rules in step 3 first: if \+any one applies, answer DIFFERENT. Only otherwise check the SAME rules in step 2: if they explain every \+difference, answer SAME. Product decisions in step 2 (e.g. different missing assets or 404 paths are one \+issue) override the intuition that each needs its own fix."""++USER_PROMPT_TEMPLATE = """Do these two error documents represent the same underlying issue?++<document_a>+{a}+</document_a>++<document_b>+{b}+</document_b>"""++VERDICT_TOOL_NAME = "record_verdict"+VERDICT_TOOL: ToolParam = {+ "name": VERDICT_TOOL_NAME,+ "description": "Record whether the two error documents are the same underlying issue.",+ "input_schema": {+ "type": "object",+ # Reasoning first, so the model names each throw site before it commits to a verdict.+ "properties": {+ "reasoning": {+ "type": "string",+ "description": "Throw site of each document, which rule decides, in two or three short sentences",+ },+ "same_issue": {+ "type": "boolean",+ "description": "True if both documents represent the same underlying issue",+ },+ },+ "required": ["reasoning", "same_issue"],+ },+}++VERDICT_TOOL_CHOICE: ToolChoiceToolParam = {"type": "tool", "name": VERDICT_TOOL_NAME}+++@frozen+class JudgeVerdict:+ outcome: JudgeOutcome+ same_issue: bool | None = None+++def _outcome_for_status(status_code: int) -> JudgeOutcome:+ if status_code >= 500 or status_code == 429:+ return "unavailable"+ return "rejected"+++def judge_pair(client: Anthropic, *, document_a: str, document_b: str, timeout_seconds: float) -> JudgeVerdict:+ try:+ # SDK retries are off so each call stays inside the caller's time budget.+ response = client.with_options(max_retries=0, timeout=timeout_seconds).messages.create(+ model=JUDGE_MODEL,+ max_tokens=MAX_TOKENS,+ system=SYSTEM_PROMPT,+ messages=[{"role": "user", "content": USER_PROMPT_TEMPLATE.format(a=document_a, b=document_b)}],+ tools=[VERDICT_TOOL],+ tool_choice=VERDICT_TOOL_CHOICE,+ extra_headers=PROVIDER_HEADERS,+ )+ except APITimeoutError:+ return JudgeVerdict(outcome="timeout")+ except APIStatusError as error:+ return JudgeVerdict(outcome=_outcome_for_status(error.status_code))+ except APIConnectionError:+ return JudgeVerdict(outcome="unavailable")+ except AnthropicError:+ return JudgeVerdict(outcome="invalid")++ for block in response.content:+ if block.type != "tool_use" or block.name != VERDICT_TOOL_NAME:+ continue+ verdict = block.input if isinstance(block.input, dict) else {}+ same_issue = verdict.get("same_issue")+ if isinstance(same_issue, bool):+ return JudgeVerdict(outcome="judged", same_issue=same_issue)+ return JudgeVerdict(outcome="invalid")diff --git a/products/error_tracking/backend/temporal/grouping_judge/sampling.py b/products/error_tracking/backend/temporal/grouping_judge/sampling.pynew file mode 100644index 000000000000..c98b1ccc152f--- /dev/null+++ b/products/error_tracking/backend/temporal/grouping_judge/sampling.py@@ -0,0 +1,50 @@+import hashlib++from posthog.dataclasses import frozen++from products.error_tracking.backend.temporal.grouping_judge.types import JudgeBand++# Fixed so bands and buckets stay comparable across changes to the live auto-merge threshold. They label+# rows for analysis and do not change the sample rate.+LOW_BAND_EDGE = 0.019+HIGH_BAND_EDGE = 0.08+DISTANCE_BUCKET_EDGES: tuple[float, ...] = (0, 0.005, 0.01, 0.019, 0.03, 0.05, 0.08)+REJUDGE_RATE = 0.1+++@frozen+class JudgeSample:+ sampled: bool+ swapped: bool+ rejudge: bool+++def band_for(distance: float) -> JudgeBand:+ if distance < LOW_BAND_EDGE:+ return "low"+ if distance < HIGH_BAND_EDGE:+ return "mid"+ return "high"+++def distance_bucket_for(distance: float) -> str:+ edges = DISTANCE_BUCKET_EDGES+ for lower, upper in zip(edges, edges[1:]):+ if distance < upper:+ return f"{lower}-{upper}"+ return f"{edges[-1]}+"+++def _unit_hash(purpose: str, team_id: int, fingerprint: str) -> float:+ """A stable value in [0, 1), so an activity retry makes the same choice for the same pair."""+ digest = hashlib.sha256(f"{purpose}:{team_id}:{fingerprint}".encode()).digest()+ return int.from_bytes(digest[:8], "big") / 2**64+++def sample_pair(team_id: int, fingerprint: str, sample_rate: float) -> JudgeSample:+ # Separate hashes per choice, so the swap and the re-judge are independent of being sampled.+ return JudgeSample(+ sampled=_unit_hash("sample", team_id, fingerprint) < sample_rate,+ swapped=_unit_hash("swap", team_id, fingerprint) < 0.5,+ rejudge=_unit_hash("rejudge", team_id, fingerprint) < REJUDGE_RATE,+ )diff --git a/products/error_tracking/backend/temporal/grouping_judge/test/test_grouping_judge.py b/products/error_tracking/backend/temporal/grouping_judge/test/test_grouping_judge.pynew file mode 100644index 000000000000..50695b38f9a8--- /dev/null+++ b/products/error_tracking/backend/temporal/grouping_judge/test/test_grouping_judge.py@@ -0,0 +1,415 @@+import threading+import dataclasses+from datetime import UTC, datetime, timedelta+from types import SimpleNamespace+from typing import Any++from posthog.test.base import BaseTest, ClickhouseTestMixin+from unittest.mock import MagicMock, patch++from django.test import SimpleTestCase++import httpx+from anthropic import APIConnectionError, APIStatusError, APITimeoutError+from parameterized import parameterized+from temporalio.exceptions import CancelledError++from posthog.clickhouse.client import sync_execute+from posthog.temporal.common.shutdown import WorkerShuttingDownError++from products.error_tracking.backend.temporal.fingerprint_embedding_result.judge import (+ DEFAULT_AUTO_MERGE_THRESHOLD,+ AutoMergeConfig,+)+from products.error_tracking.backend.temporal.fingerprint_embedding_result.types import (+ AutoMergeOutcome,+ FingerprintEmbeddingResultInputs,+ SimilarFingerprintDistance,+)+from products.error_tracking.backend.temporal.grouping_judge.algorithms import (+ EMBEDDING_THRESHOLD_V1_DISTANCE,+ LIVE_ALGORITHM,+ decisions_for,+)+from products.error_tracking.backend.temporal.grouping_judge.config import (+ JUDGE_OFF,+ JudgeConfig,+ current_judge_config,+ parse_judge_config,+)+from products.error_tracking.backend.temporal.grouping_judge.inline import (+ _fetch_document_texts,+ grouping_judge_properties,+)+from products.error_tracking.backend.temporal.grouping_judge.judge import judge_pair+from products.error_tracking.backend.temporal.grouping_judge.sampling import band_for, distance_bucket_for, sample_pair++MODULE = "products.error_tracking.backend.temporal.grouping_judge"+REQUEST = httpx.Request("POST", "http://gateway.example.com/v1/messages")+NEW_TEXT = "TypeError: secret customer message\n at checkout (app.js)"+NEIGHBOUR_TEXT = "TypeError: other customer message\n at checkout (app.js)"+ALWAYS_JUDGE = JudgeConfig(sample_rate=1.0)+NO_MERGE = AutoMergeOutcome()+++def _inputs(team_id: int = 1) -> FingerprintEmbeddingResultInputs:+ return FingerprintEmbeddingResultInputs(+ team_id=team_id,+ fingerprint="fp-new",+ rendering="type_message_and_stack",+ timestamp="2026-10-14T09:00:00Z",+ model_name="text-embedding-3-large-3072",+ embedding=[0.1],+ )+++def _tool_response(same_issue: Any, reasoning: str = "same frames") -> SimpleNamespace:+ return SimpleNamespace(+ content=[+ SimpleNamespace(+ type="tool_use", name="record_verdict", input={"same_issue": same_issue, "reasoning": reasoning}+ )+ ]+ )+++def _client_returning(response: Any = None, error: Exception | None = None) -> MagicMock:+ client = MagicMock()+ create = client.with_options.return_value.messages.create+ if error is not None:+ create.side_effect = error+ else:+ create.return_value = response+ return client+++class TestAlgorithms(SimpleTestCase):+ @parameterized.expand([("below", 0.0189, "merge"), ("at_threshold", 0.019, "no_merge")])+ def test_live_algorithm_decides_from_distance(self, _name: str, distance: float, expected: str) -> None:+ assert decisions_for(distance) == {LIVE_ALGORITHM.id: expected}++ def test_v1_matches_the_live_auto_merge_threshold(self) -> None:+ assert EMBEDDING_THRESHOLD_V1_DISTANCE == DEFAULT_AUTO_MERGE_THRESHOLD+++class TestSampling(SimpleTestCase):+ @parameterized.expand(+ [+ ("below_low_edge", 0.0189, "low", "0.01-0.019"),+ ("at_low_edge", 0.019, "mid", "0.019-0.03"),+ ("at_high_edge", 0.08, "high", "0.08+"),+ ]+ )+ def test_band_and_bucket_edges(self, _name: str, distance: float, band: str, bucket: str) -> None:+ assert (band_for(distance), distance_bucket_for(distance)) == (band, bucket)++ def test_sampling_is_stable_across_retries(self) -> None:+ first = [sample_pair(1, f"fp-{index}", 0.5) for index in range(200)]+ again = [sample_pair(1, f"fp-{index}", 0.5) for index in range(200)]++ assert first == again+ assert 0 < sum(sample.sampled for sample in first) < 200+++class TestJudgeConfig(SimpleTestCase):+ @parameterized.expand(+ [+ (+ "this_region",+ {"EU": {"sample_rate": 0.01, "team_ids": [2]}},+ JudgeConfig(sample_rate=0.01, team_ids=frozenset({2})),+ ),+ ("json_string", '{"EU": {"sample_rate": 0.01}}', JudgeConfig(sample_rate=0.01)),+ ("other_region_only", {"US": {"sample_rate": 0.01}}, JUDGE_OFF),+ ("flag_missing_or_fetch_failed", None, JUDGE_OFF),+ ("rate_above_one", {"EU": {"sample_rate": 5}}, JUDGE_OFF),+ ("rate_as_string", {"EU": {"sample_rate": "0.01"}}, JUDGE_OFF),+ ("team_ids_as_strings", {"EU": {"sample_rate": 0.01, "team_ids": ["2"]}}, JUDGE_OFF),+ ("not_json", "{oops", JUDGE_OFF),+ ]+ )+ def test_anything_but_a_valid_entry_for_this_region_is_off(+ self, _name: str, payload: object, expected: JudgeConfig+ ) -> None:+ assert parse_judge_config(payload, "EU") == expected++ def test_off_without_reading_the_flag_when_no_definitions_are_loaded(self) -> None:+ with (+ patch(f"{MODULE}.config.posthoganalytics.feature_flag_definitions", return_value=None),+ patch(f"{MODULE}.config.posthoganalytics.get_feature_flag_payload") as get_payload,+ ):+ assert current_judge_config() == JUDGE_OFF++ get_payload.assert_not_called()+++class TestJudgeOutcome(SimpleTestCase):+ @parameterized.expand(+ [+ (+ "server_error",+ APIStatusError("down", response=httpx.Response(503, request=REQUEST), body=None),+ "unavailable",+ ),+ (+ "rate_limited",+ APIStatusError("slow", response=httpx.Response(429, request=REQUEST), body=None),+ "unavailable",+ ),+ ("transport", APIConnectionError(request=REQUEST), "unavailable"),+ ("timeout", APITimeoutError(request=REQUEST), "timeout"),+ (+ "client_error",+ APIStatusError("bad", response=httpx.Response(400, request=REQUEST), body=None),+ "rejected",+ ),+ ]+ )+ def test_errors_are_not_verdicts(self, _name: str, error: Exception, outcome: str) -> None:+ verdict = judge_pair(_client_returning(error=error), document_a="a", document_b="b", timeout_seconds=5)++ assert (verdict.outcome, verdict.same_issue) == (outcome, None)++ @parameterized.expand(+ [+ ("non_bool_answer", _tool_response("yes"), "invalid", None),+ ("no_tool_call", SimpleNamespace(content=[SimpleNamespace(type="text", text="yes")]), "invalid", None),+ ("tool_answer", _tool_response(False), "judged", False),+ ]+ )+ def test_answer_parsing(self, _name: str, response: SimpleNamespace, outcome: str, same_issue: bool | None) -> None:+ verdict = judge_pair(_client_returning(response), document_a="a", document_b="b", timeout_seconds=5)++ assert (verdict.outcome, verdict.same_issue) == (outcome, same_issue)+++class TestGroupingJudgeProperties(SimpleTestCase):+ def _run(+ self,+ client: MagicMock,+ *,+ approved: bool | None = True,+ text_rows: list[tuple[str, str]] | None = None,+ rejudge_rate: float = 0.0,+ merge_outcome: AutoMergeOutcome | None = NO_MERGE,+ config: JudgeConfig = ALWAYS_JUDGE,+ input_content: str | None = None,+ closest: list[SimilarFingerprintDistance] | None = None,+ ) -> tuple[dict[str, object], MagicMock, MagicMock]:+ team = MagicMock(id=1)+ team.organization.is_ai_data_processing_approved = approved+ rows = text_rows if text_rows is not None else [("fp-new", NEW_TEXT), ("fp-old", NEIGHBOUR_TEXT)]+ with (+ patch(f"{MODULE}.inline.execute_hogql_query", return_value=MagicMock(results=rows)) as hogql,+ patch(f"{MODULE}.inline.build_ai_gateway_anthropic_client", return_value=client) as build_client,+ patch(f"{MODULE}.sampling.REJUDGE_RATE", rejudge_rate),+ patch(f"{MODULE}.inline.current_judge_config", return_value=config),+ ):+ properties = grouping_judge_properties(+ team,+ dataclasses.replace(_inputs(), content=input_content),+ closest or [SimilarFingerprintDistance(fingerprint="fp-old", distance=0.01)],+ merge_outcome,+ )+ return properties, hogql, build_client++ @parameterized.expand(+ [+ ("disabled", JUDGE_OFF, NO_MERGE),+ ("team_not_listed", JudgeConfig(sample_rate=1.0, team_ids=frozenset({2})), NO_MERGE),+ ("merge_raised", ALWAYS_JUDGE, None),+ ]+ )+ def test_off_makes_no_text_fetch_and_no_call(+ self, _name: str, config: JudgeConfig, merge_outcome: AutoMergeOutcome | None+ ) -> None:+ client = _client_returning(_tool_response(True))++ properties, hogql, build_client = self._run(client, merge_outcome=merge_outcome, config=config)++ assert properties["judge_sampled"] is False+ assert "judge_outcome" not in properties+ hogql.assert_not_called()+ build_client.assert_not_called()++ @parameterized.expand([("not_approved", False), ("unset", None)])+ def test_team_without_ai_consent_is_not_judged(self, _name: str, approved: bool | None) -> None:+ client = _client_returning(_tool_response(True))++ properties, hogql, _ = self._run(client, approved=approved)++ assert properties["judge_outcome"] == "no_consent"+ hogql.assert_not_called()+ client.with_options.return_value.messages.create.assert_not_called()++ @parameterized.expand(+ [+ ("neighbour_missing", [("fp-new", NEW_TEXT)], NEW_TEXT, "no_text"),+ ("new_text_missing_and_not_in_input", [("fp-old", NEIGHBOUR_TEXT)], None, "no_text"),+ ("new_text_not_in_clickhouse_yet", [("fp-old", NEIGHBOUR_TEXT)], NEW_TEXT, "judged"),+ ]+ )+ def test_judges_only_with_both_texts(+ self, _name: str, text_rows: list[tuple[str, str]], input_content: str | None, outcome: str+ ) -> None:+ client = _client_returning(_tool_response(True))++ properties, _, _ = self._run(client, text_rows=text_rows, input_content=input_content)++ create = client.with_options.return_value.messages.create+ assert properties["judge_outcome"] == outcome+ assert create.called is (outcome == "judged")+ if create.called:+ assert "secret customer" in create.call_args.kwargs["messages"][0]["content"]++ def test_rejudge_swaps_order_and_event_carries_no_text_or_reasoning(self) -> None:+ client = _client_returning(_tool_response(True, reasoning="REASONING-MARKER"))++ properties, _, _ = self._run(+ client,+ rejudge_rate=1.0,+ merge_outcome=AutoMergeOutcome(merged_count=1, target_fingerprint="fp-old"),+ )++ first_prompt, second_prompt = (+ call.kwargs["messages"][0]["content"]+ for call in client.with_options.return_value.messages.create.call_args_list+ )+ new_first_in_first_prompt = first_prompt.index("secret customer") < first_prompt.index("other customer")+ assert new_first_in_first_prompt is not properties["judge_swapped"]+ assert new_first_in_first_prompt != (+ second_prompt.index("secret customer") < second_prompt.index("other customer")+ )+ assert {key: value for key, value in properties.items() if key != "judge_duration_ms"} == {+ "live_decision": "merge",+ "live_target_fingerprint": "fp-old",+ "live_merge_decision": None,+ "live_algorithm": "embedding_threshold_v1",+ "live_auto_merge_threshold": 0.019,+ "live_judge_enabled": False,+ "live_judge_shadow": True,+ "live_judge_cutoff": 0.9,+ "live_judge_max_distance": 0.1,+ "decisions": {"embedding_threshold_v1": "merge"},+ "judge_band": "low",+ "distance_bucket": "0.01-0.019",+ "judge_sampled": True,+ "judge_target_fingerprint": "fp-old",+ "judge_outcome": "judged",+ "judge_same_issue": True,+ "judge_swapped": properties["judge_swapped"],+ "judge_rejudge_same_issue": True,+ "judge_model": "claude-haiku-4-5",+ "judge_view": "type_message_and_stack",+ }+ serialized = repr(properties)+ for leaked in ("customer message", "app.js", "REASONING-MARKER"):+ assert leaked not in serialized++ def test_judges_the_pair_the_merge_used_when_it_skipped_rank_one(self) -> None:+ client = _client_returning(_tool_response(True))++ properties, _, _ = self._run(+ client,+ text_rows=[("fp-new", NEW_TEXT), ("fp-old", NEIGHBOUR_TEXT), ("fp-merged", "TypeError: merged target")],+ merge_outcome=AutoMergeOutcome(merged_count=1, target_fingerprint="fp-merged"),+ closest=[+ SimilarFingerprintDistance(fingerprint="fp-old", distance=0.01),+ SimilarFingerprintDistance(fingerprint="fp-merged", distance=0.015),+ ],+ )++ prompt = client.with_options.return_value.messages.create.call_args.kwargs["messages"][0]["content"]+ assert properties["judge_target_fingerprint"] == "fp-merged"+ assert "merged target" in prompt+ assert "other customer" not in prompt++ def test_long_documents_keep_their_throw_site(self) -> None:+ client = _client_returning(_tool_response(True))+ long_stack = "TypeError: boom\n" + " at outer_caller (app.js)\n" * 500 + " at THROW_SITE (app.js)\n"++ self._run(client, input_content=long_stack)++ prompt = client.with_options.return_value.messages.create.call_args.kwargs["messages"][0]["content"]+ assert "TypeError: boom" in prompt+ assert "THROW_SITE" in prompt++ def test_records_the_merge_config_that_made_the_decision(self) -> None:+ merge_config = AutoMergeConfig(judge_enabled=True, judge_shadow=False)++ with patch(f"{MODULE}.inline.current_judge_config", return_value=JUDGE_OFF):+ properties = grouping_judge_properties(+ MagicMock(id=1),+ _inputs(),+ [SimilarFingerprintDistance(fingerprint="fp-old", distance=0.05)],+ AutoMergeOutcome(merged_count=1, target_fingerprint="fp-old", merge_decision="judge"),+ merge_config,+ )++ assert properties["live_algorithm"] == "auto_merge_config"+ assert properties["live_merge_decision"] == "judge"+ assert (properties["live_judge_enabled"], properties["live_judge_shadow"]) == (True, False)++ @parameterized.expand(+ [+ ("temporal_cancel", CancelledError("Cancelled")),+ ("worker_shutdown", WorkerShuttingDownError("activity-1", "merge", "queue", 1, None, None)),+ ]+ )+ def test_a_drain_during_the_judge_ends_the_judge_not_the_activity(self, _name: str, error: Exception) -> None:+ properties, _, _ = self._run(_client_returning(error=error))++ assert properties["judge_outcome"] == "cancelled"++ def test_skips_the_judge_when_every_worker_is_busy(self) -> None:+ client = _client_returning(_tool_response(True))+ no_free_workers = threading.BoundedSemaphore(1)+ no_free_workers.acquire()++ with patch(f"{MODULE}.inline._free_workers", no_free_workers):+ properties, hogql, _ = self._run(client)++ assert properties["judge_outcome"] == "unavailable"+ hogql.assert_not_called()+++class TestFetchDocumentTexts(ClickhouseTestMixin, BaseTest):+ def _insert(self, team_id: int, document_id: str, content: str, timestamp: datetime, inserted_at: datetime) -> None:+ sync_execute(+ """+ INSERT INTO distributed_posthog_document_embeddings_text_embedding_3_large_3072 (+ team_id, product, document_type, rendering, document_id,+ timestamp, inserted_at, content, metadata, embedding,+ _timestamp, _offset, _partition+ ) VALUES+ """,+ [+ (+ team_id,+ "error_tracking",+ "fingerprint",+ "type_message_and_stack",+ document_id,+ timestamp,+ inserted_at,+ content,+ "{}",+ [0.0] * 3072,+ inserted_at,+ 0,+ 0,+ )+ ],+ flush=False,+ team_id=team_id,+ )++ def test_reads_latest_text_for_this_team_only(self) -> None:+ created = datetime(2026, 10, 14, 9, 0, tzinfo=UTC)+ self._insert(self.team.id, "fp-new", "old render", created, created)+ self._insert(self.team.id, "fp-new", "x" * 7000, created, created + timedelta(minutes=1))+ self._insert(self.team.id + 1, "fp-old", "other team text", created - timedelta(days=1), created)++ texts = _fetch_document_texts(self.team, _inputs(self.team.id), "fp-old")++ assert texts == {"fp-new": "x" * 7000}diff --git a/products/error_tracking/backend/temporal/grouping_judge/types.py b/products/error_tracking/backend/temporal/grouping_judge/types.pynew file mode 100644index 000000000000..c3e31a7c1e97--- /dev/null+++ b/products/error_tracking/backend/temporal/grouping_judge/types.py@@ -0,0 +1,6 @@+from typing import Literal++Decision = Literal["merge", "no_merge"]+LiveDecision = Literal["merge", "no_merge", "error"]+JudgeOutcome = Literal["judged", "unavailable", "rejected", "invalid", "timeout", "cancelled", "no_text", "no_consent"]+JudgeBand = Literal["low", "mid", "high"]diff --git a/products/error_tracking/backend/temporal/lifecycle/issue_created/test/test_workflow.py b/products/error_tracking/backend/temporal/lifecycle/issue_created/test/test_workflow.pyindex ab32e000cbbc..99198c834432 100644--- a/products/error_tracking/backend/temporal/lifecycle/issue_created/test/test_workflow.py+++ b/products/error_tracking/backend/temporal/lifecycle/issue_created/test/test_workflow.py@@ -207,6 +207,27 @@ def test_embedding_failure_classification_and_capture(assert capture_exception.called is expect_captured+@patch("products.error_tracking.backend.temporal.lifecycle.issue_created.activities.render_stacktrace")+@patch("products.error_tracking.backend.temporal.lifecycle.issue_created.activities.fetch_event_properties")+@patch("products.error_tracking.backend.temporal.lifecycle.issue_created.activities.Team.objects.get")+@patch("products.error_tracking.backend.temporal.lifecycle.issue_created.activities.generate_embedding")+def test_embedding_hands_the_judge_its_text_before_clickhouse_has_it(+ generate_embedding: MagicMock,+ get_team: MagicMock,+ fetch_event_properties: MagicMock,+ render_stacktrace: MagicMock,+) -> None:+ get_team.return_value.organization.is_ai_data_processing_approved = True+ fetch_event_properties.return_value = {"$exception_list": [{"type": "TypeError", "value": "boom"}]}+ render_stacktrace.return_value = "TypeError: boom"+ generate_embedding.return_value = MagicMock(embedding=[0.1, 0.2])++ result = generate_issue_created_embedding_activity(_inputs("fingerprint"))++ assert result.embedding is not None+ assert result.embedding.merge_inputs.content == "TypeError: boom"++def _decision(choice: str) -> DecisionResult:return DecisionResult(model="severity-model",