Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(error-tracking): judge a sample of auto-merge decisions inline (#117087)

James Gill2b1551e3e844
diff --git a/products/error_tracking/backend/temporal/fingerprint_embedding_result/activities.py b/products/error_tracking/backend/temporal/fingerprint_embedding_result/activities.py
index 739892089357..026e138a2188 100644
--- a/products/error_tracking/backend/temporal/fingerprint_embedding_result/activities.py
+++ b/products/error_tracking/backend/temporal/fingerprint_embedding_result/activities.py
@@ -1,10 +1,8 @@
import math
import time
import uuid
-from datetime import datetime, timedelta
from django.conf import settings
-from django.utils.dateparse import parse_datetime
from posthog.hogql import ast
from posthog.hogql.constants import HogQLGlobalSettings
@@ -36,12 +34,15 @@ from products.error_tracking.backend.temporal.fingerprint_embedding_result.judge
load_auto_merge_config,
)
from products.error_tracking.backend.temporal.fingerprint_embedding_result.types import (
+ CLOSEST_FINGERPRINTS_WINDOW,
AutoMergeOutcome,
AutoMergeReopenedTarget,
FingerprintEmbeddingMergeResult,
FingerprintEmbeddingResultInputs,
SimilarFingerprintDistance,
+ parse_embedding_timestamp,
)
+from products.error_tracking.backend.temporal.grouping_judge.inline import grouping_judge_properties
from products.error_tracking.backend.temporal.lifecycle.types import LifecycleIssueSnapshot
NIL_UUID = "00000000-0000-0000-0000-000000000000"
@@ -147,19 +148,12 @@ def _model_specific_query(query_by_model: dict[str, str], model_name: str) -> st
return query
-def _parse_timestamp(timestamp: str) -> datetime:
- parsed = parse_datetime(timestamp)
- if parsed is None:
- raise ValueError(f"Invalid embedding timestamp: {timestamp}")
- return parsed
-
-
def _closest_fingerprints_query(
inputs: FingerprintEmbeddingResultInputs,
model_name: str,
target_embedding: list[float],
) -> ast.SelectQuery | ast.SelectSetQuery:
- min_timestamp = _parse_timestamp(inputs.timestamp) - timedelta(days=30)
+ min_timestamp = parse_embedding_timestamp(inputs.timestamp) - CLOSEST_FINGERPRINTS_WINDOW
return parse_select(
_model_specific_query(CLOSEST_FINGERPRINTS_QUERY_BY_MODEL, model_name),
placeholders={
@@ -212,6 +206,7 @@ def _report_closest_fingerprint_metrics(
closest_fingerprints: list[SimilarFingerprintDistance],
model_name: str,
query_duration_ms: float,
+ judge_properties: dict[str, object],
) -> None:
properties: dict[str, object] = {
"model_name": model_name,
@@ -219,6 +214,7 @@ def _report_closest_fingerprint_metrics(
"fingerprint": inputs.fingerprint,
"query_duration_ms": query_duration_ms,
"closest_fingerprint_count": len(closest_fingerprints),
+ **judge_properties,
}
for index, closest_fingerprint in enumerate(closest_fingerprints[:3], start=1):
properties[f"rank_{index}_fingerprint"] = closest_fingerprint.fingerprint
@@ -309,7 +305,12 @@ def _merge_into_candidate(
reopened_target = (
_reopened_target(target_issue, event_reference=event_reference) if merge_outcome.reopened else None
)
- return AutoMergeOutcome(merged_count=1, reopened_target=reopened_target)
+ return AutoMergeOutcome(
+ merged_count=1,
+ reopened_target=reopened_target,
+ target_fingerprint=candidate.fingerprint,
+ merge_decision=str(decision_properties["merge_decision"]),
+ )
def _fetch_fingerprint_document(
@@ -322,7 +323,9 @@ def _fetch_fingerprint_document(
"rendering": ast.Constant(value=inputs.rendering),
"team_id": ast.Constant(value=team.id),
"fingerprint": ast.Constant(value=fingerprint),
- "min_timestamp": ast.Constant(value=_parse_timestamp(inputs.timestamp) - timedelta(days=30)),
+ "min_timestamp": ast.Constant(
+ value=parse_embedding_timestamp(inputs.timestamp) - CLOSEST_FINGERPRINTS_WINDOW
+ ),
},
)
response = execute_hogql_query(
@@ -505,20 +508,30 @@ def merge_similar_fingerprints(
query_duration_seconds = time.monotonic() - start
query_duration_ms = query_duration_seconds * 1000
- # Keep emitting candidate metrics for every run; merging is gated separately by configuration.
- _report_closest_fingerprint_metrics(team, inputs, closest_fingerprints, inputs.model_name, query_duration_ms)
- merge_outcome = _merge_fingerprint_into_closest_issue(
- team,
- inputs.fingerprint,
- closest_fingerprints,
- inputs.source_issue_id,
- event_reference=_event_reference(inputs),
- # Only read the flag when there is something to decide.
- config=load_auto_merge_config(team)
+ # Only read the flag when there is something to decide.
+ config = (
+ load_auto_merge_config(team)
if settings.ERROR_TRACKING_AUTO_MERGE_ENABLED and closest_fingerprints
- else DEFAULT_AUTO_MERGE_CONFIG,
- judge_inputs=inputs,
+ else DEFAULT_AUTO_MERGE_CONFIG
)
+ merge_outcome: AutoMergeOutcome | None = None
+ try:
+ merge_outcome = _merge_fingerprint_into_closest_issue(
+ team,
+ inputs.fingerprint,
+ closest_fingerprints,
+ inputs.source_issue_id,
+ event_reference=_event_reference(inputs),
+ config=config,
+ judge_inputs=inputs,
+ )
+ finally:
+ # Keep emitting candidate metrics for every run; merging is gated separately by configuration.
+ # The grouping judge runs only after the merge decision, so it cannot change it.
+ judge_properties = grouping_judge_properties(team, inputs, closest_fingerprints, merge_outcome, config)
+ _report_closest_fingerprint_metrics(
+ team, inputs, closest_fingerprints, inputs.model_name, query_duration_ms, judge_properties
+ )
return FingerprintEmbeddingMergeResult(
merged_count=merge_outcome.merged_count,
diff --git a/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_fingerprint_embedding_result.py b/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_fingerprint_embedding_result.py
index b9a37e9a7bf1..d88fb1f2a687 100644
--- a/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_fingerprint_embedding_result.py
+++ b/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_fingerprint_embedding_result.py
@@ -1,6 +1,8 @@
import uuid
+import threading
from datetime import UTC, datetime
-from typing import cast
+from types import SimpleNamespace
+from typing import Any, cast
import pytest
from posthog.test.base import BaseTest
@@ -9,6 +11,9 @@ from unittest.mock import MagicMock, patch
from django.db import OperationalError
from django.test import override_settings
+import httpx
+from anthropic import APIConnectionError
+from parameterized import parameterized
from temporalio.exceptions import CancelledError
from posthog.clickhouse.client.connection import ClickHouseUser, Workload
@@ -38,10 +43,13 @@ from products.error_tracking.backend.temporal.fingerprint_embedding_result.types
FingerprintEmbeddingResultInputs,
SimilarFingerprintDistance,
)
+from products.error_tracking.backend.temporal.grouping_judge.config import JudgeConfig
from products.error_tracking.backend.temporal.lifecycle.issue_created.activities import (
merge_issue_created_fingerprint_activity,
)
+JUDGE_MODULE = "products.error_tracking.backend.temporal.grouping_judge.inline"
+
def _inputs() -> FingerprintEmbeddingResultInputs:
return FingerprintEmbeddingResultInputs(
@@ -139,6 +147,7 @@ class TestFingerprintEmbeddingResultActivity:
closest_fingerprints=closest_fingerprints,
model_name="text-embedding-3-large-3072",
query_duration_ms=12.3,
+ judge_properties={"live_decision": "merge"},
)
properties = capture.call_args.kwargs["properties"]
@@ -148,6 +157,7 @@ class TestFingerprintEmbeddingResultActivity:
assert properties["rank_1_distance"] == 0.01
assert properties["rank_2_fingerprint"] == "fingerprint-2"
assert properties["rank_3_fingerprint"] == "fingerprint-3"
+ assert properties["live_decision"] == "merge"
def test_merge_activity_reports_distances(self) -> None:
closest_fingerprints = [
@@ -218,6 +228,44 @@ class TestFingerprintEmbeddingResultActivity:
assert capture_activity_exception.called is expect_captured
+ def test_merge_activity_reports_an_error_decision_and_reraises_when_the_merge_raises(self) -> None:
+ error = StaleAutoMergeStateError("ownership changed")
+ capture = MagicMock()
+
+ with (
+ patch(
+ "products.error_tracking.backend.temporal.fingerprint_embedding_result.activities.Team.objects.get",
+ return_value=MagicMock(id=1, uuid=uuid.uuid4()),
+ ),
+ patch(
+ "products.error_tracking.backend.temporal.fingerprint_embedding_result.activities._query_closest_fingerprints",
+ return_value=[SimilarFingerprintDistance(fingerprint="fingerprint-1", distance=0.01)],
+ ),
+ patch(
+ "products.error_tracking.backend.temporal.fingerprint_embedding_result.activities._merge_fingerprint_into_closest_issue",
+ side_effect=error,
+ ),
+ patch(
+ "products.error_tracking.backend.temporal.fingerprint_embedding_result.activities.ph_background_capture",
+ return_value=capture,
+ ),
+ patch(
+ "products.error_tracking.backend.temporal.fingerprint_embedding_result.activities.groups",
+ return_value={},
+ ),
+ patch(
+ "products.error_tracking.backend.temporal.fingerprint_embedding_result.activities._capture_activity_exception"
+ ),
+ pytest.raises(StaleAutoMergeStateError) as raised,
+ ):
+ merge_issue_created_fingerprint_activity(_inputs())
+
+ assert raised.value is error
+ (call,) = capture.call_args_list
+ assert call.kwargs["event"] == "error_tracking_fingerprint_embedding_result_metrics"
+ assert call.kwargs["properties"]["live_decision"] == "error"
+ assert call.kwargs["properties"]["live_target_fingerprint"] is None
+
def test_merge_fingerprint_skips_when_auto_merge_disabled(self) -> None:
with override_settings(ERROR_TRACKING_AUTO_MERGE_ENABLED=False):
result = _merge_fingerprint_into_closest_issue(
@@ -566,3 +614,105 @@ class TestAutoMergeReopensTarget(BaseTest):
assert outcome.merged_count == 1
assert outcome.reopened_target is not None
assert outcome.reopened_target.issue_id == str(target.id)
+
+
+@override_settings(ERROR_TRACKING_AUTO_MERGE_ENABLED=True)
+class TestGroupingJudgeNeverChangesTheMerge(BaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ self.organization.is_ai_data_processing_approved = True
+ self.organization.save()
+ for fingerprint in ("fp-source", "fp-target"):
+ issue = ErrorTrackingIssue.objects.create(team=self.team, name="TypeError")
+ ErrorTrackingIssueFingerprintV2.objects.create(team=self.team, issue=issue, fingerprint=fingerprint)
+ self.target_issue_id = ErrorTrackingIssueFingerprintV2.objects.get(fingerprint="fp-target").issue_id
+
+ def _run(self, client: MagicMock | None = None, build_error: Exception | None = None) -> tuple[Any, list[Any]]:
+ inputs = FingerprintEmbeddingResultInputs(
+ team_id=self.team.id,
+ fingerprint="fp-source",
+ rendering="type_message_and_stack",
+ timestamp="2026-06-08T00:00:00Z",
+ model_name="text-embedding-3-large-3072",
+ embedding=[0.1],
+ )
+ texts = MagicMock(results=[("fp-source", "TypeError: a"), ("fp-target", "TypeError: b")])
+ capture = MagicMock()
+ with (
+ patch(
+ "products.error_tracking.backend.temporal.fingerprint_embedding_result.activities._query_closest_fingerprints",
+ return_value=[SimilarFingerprintDistance(fingerprint="fp-target", distance=0.01)],
+ ),
+ patch(
+ "products.error_tracking.backend.temporal.fingerprint_embedding_result.activities.ph_background_capture",
+ return_value=capture,
+ ),
+ patch(f"{JUDGE_MODULE}.execute_hogql_query", return_value=texts),
+ patch(f"{JUDGE_MODULE}.build_ai_gateway_anthropic_client", return_value=client, side_effect=build_error),
+ patch(f"{JUDGE_MODULE}.TOTAL_BUDGET_SECONDS", 0.5),
+ patch(f"{JUDGE_MODULE}.current_judge_config", return_value=JudgeConfig(sample_rate=1.0)),
+ ):
+ result = merge_issue_created_fingerprint_activity(inputs)
+ events = [
+ call.kwargs["properties"]
+ for call in capture.call_args_list
+ if call.kwargs["event"] == "error_tracking_fingerprint_embedding_result_metrics"
+ ]
+ return result, events
+
+ @parameterized.expand(
+ [
+ ("judged", "judged"),
+ ("judge_raises", "unavailable"),
+ ("gateway_not_configured", "unavailable"),
+ ("gateway_down", "unavailable"),
+ ("judge_hangs", "timeout"),
+ ("worker_drains_during_the_judge", "cancelled"),
+ ]
+ )
+ def test_merge_and_event_survive_any_judge_failure(self, case: str, outcome: str) -> None:
+ release = threading.Event()
+ client = MagicMock()
+ create = client.with_options.return_value.messages.create
+ build_error: Exception | None = None
+ if case == "judged":
+ create.return_value = SimpleNamespace(
+ content=[SimpleNamespace(type="tool_use", name="record_verdict", input={"same_issue": True})]
+ )
+ elif case == "judge_raises":
+ build_error = RuntimeError("judge bug")
+ elif case == "gateway_not_configured":
+ build_error = ValueError("AI_GATEWAY_URL and AI_GATEWAY_API_KEY must be configured")
+ elif case == "gateway_down":
+ create.side_effect = APIConnectionError(request=httpx.Request("POST", "http://gateway.example.com"))
+ elif case == "worker_drains_during_the_judge":
+ create.side_effect = CancelledError("Cancelled")
+ else:
+ create.side_effect = lambda **_: release.wait(timeout=5)
+
+ try:
+ result, events = self._run(client, build_error)
+ finally:
+ release.set()
+
+ assert (result.merged_count, result.reopened_target, result.closest_fingerprints) == (
+ 1,
+ None,
+ [SimilarFingerprintDistance(fingerprint="fp-target", distance=0.01)],
+ )
+ assert ErrorTrackingIssueFingerprintV2.objects.get(fingerprint="fp-source").issue_id == self.target_issue_id
+ (event,) = events
+ assert (
+ event["live_decision"],
+ event["live_target_fingerprint"],
+ event["live_merge_decision"],
+ event["judge_outcome"],
+ ) == ("merge", "fp-target", "distance", outcome)
+
+ def test_merge_and_event_survive_a_failure_outside_the_judge_call(self) -> None:
+ with patch(f"{JUDGE_MODULE}.band_for", side_effect=RuntimeError("judge bug")):
+ result, events = self._run()
+
+ assert result.merged_count == 1
+ assert ErrorTrackingIssueFingerprintV2.objects.get(fingerprint="fp-source").issue_id == self.target_issue_id
+ assert [event["live_decision"] for event in events] == ["merge"]
diff --git a/products/error_tracking/backend/temporal/fingerprint_embedding_result/types.py b/products/error_tracking/backend/temporal/fingerprint_embedding_result/types.py
index 9c32b7968285..9e01bda7b4dd 100644
--- a/products/error_tracking/backend/temporal/fingerprint_embedding_result/types.py
+++ b/products/error_tracking/backend/temporal/fingerprint_embedding_result/types.py
@@ -1,7 +1,19 @@
import dataclasses
+from datetime import datetime, timedelta
+
+from django.utils.dateparse import parse_datetime
from products.error_tracking.backend.temporal.lifecycle.types import LifecycleIssueSnapshot
+CLOSEST_FINGERPRINTS_WINDOW = timedelta(days=30)
+
+
+def parse_embedding_timestamp(timestamp: str) -> datetime:
+ parsed = parse_datetime(timestamp)
+ if parsed is None:
+ raise ValueError(f"Invalid embedding timestamp: {timestamp}")
+ return parsed
+
@dataclasses.dataclass(frozen=True)
class FingerprintEmbeddingResultInputs:
@@ -38,6 +50,9 @@ class AutoMergeReopenedTarget:
class AutoMergeOutcome:
merged_count: int = 0
reopened_target: AutoMergeReopenedTarget | None = None
+ target_fingerprint: str | None = None
+ # Which rule made the merge: "distance" or "judge". None when nothing merged in this attempt.
+ merge_decision: str | None = None
@dataclasses.dataclass(frozen=True)
diff --git a/products/error_tracking/backend/temporal/grouping_judge/algorithms.py b/products/error_tracking/backend/temporal/grouping_judge/algorithms.py
new file mode 100644
index 000000000000..73a46a12c4dc
--- /dev/null
+++ b/products/error_tracking/backend/temporal/grouping_judge/algorithms.py
@@ -0,0 +1,29 @@
+from collections.abc import Callable
+
+from posthog.dataclasses import frozen
+
+from products.error_tracking.backend.temporal.grouping_judge.types import Decision
+
+
+@frozen
+class GroupingAlgorithm:
+ id: str
+ decide: Callable[[float], Decision]
+
+
+# A fixed copy of the live threshold. A change to the live threshold is a new algorithm, so rows
+# recorded under this id stay comparable.
+EMBEDDING_THRESHOLD_V1_DISTANCE = 0.019
+
+EMBEDDING_THRESHOLD_V1 = GroupingAlgorithm(
+ id="embedding_threshold_v1",
+ decide=lambda distance: "merge" if distance < EMBEDDING_THRESHOLD_V1_DISTANCE else "no_merge",
+)
+
+ALGORITHMS: tuple[GroupingAlgorithm, ...] = (EMBEDDING_THRESHOLD_V1,)
+# The algorithm production runs. Point this at another entry when production switches algorithm.
+LIVE_ALGORITHM = EMBEDDING_THRESHOLD_V1
+
+
+def decisions_for(distance: float, algorithms: tuple[GroupingAlgorithm, ...] = ALGORITHMS) -> dict[str, Decision]:
+ return {algorithm.id: algorithm.decide(distance) for algorithm in algorithms}
diff --git a/products/error_tracking/backend/temporal/grouping_judge/config.py b/products/error_tracking/backend/temporal/grouping_judge/config.py
new file mode 100644
index 000000000000..b59268cee11d
--- /dev/null
+++ b/products/error_tracking/backend/temporal/grouping_judge/config.py
@@ -0,0 +1,71 @@
+import json
+
+import structlog
+import posthoganalytics
+
+from posthog.dataclasses import frozen
+from posthog.utils import get_instance_region
+
+logger = structlog.get_logger(__name__)
+
+REMOTE_CONFIG_KEY = "error-tracking-grouping-judge"
+REMOTE_CONFIG_DISTINCT_ID = "error_tracking_grouping_judge"
+
+
+@frozen
+class JudgeConfig:
+ sample_rate: float = 0.0
+ # Empty means every team.
+ team_ids: frozenset[int] = frozenset()
+
+ def is_enabled_for(self, team_id: int) -> bool:
+ return self.sample_rate > 0 and (not self.team_ids or team_id in self.team_ids)
+
+
+JUDGE_OFF = JudgeConfig()
+
+
+def parse_judge_config(payload: object, region: str | None) -> JudgeConfig:
+ """Read this region's entry, e.g. `{"EU": {"sample_rate": 0.01, "team_ids": []}}`.
+
+ Team ids repeat across regions, so each region has its own entry. Anything missing or malformed is off.
+ """
+ if isinstance(payload, str):
+ try:
+ payload = json.loads(payload)
+ except ValueError:
+ return JUDGE_OFF
+ if not isinstance(payload, dict) or region is None:
+ return JUDGE_OFF
+ entry = payload.get(region)
+ if not isinstance(entry, dict):
+ return JUDGE_OFF
+ sample_rate = entry.get("sample_rate", 0)
+ team_ids = entry.get("team_ids", [])
+ if (
+ isinstance(sample_rate, bool)
+ or not isinstance(sample_rate, int | float)
+ or not 0 <= sample_rate <= 1
+ or not isinstance(team_ids, list)
+ or not all(isinstance(team_id, int) and not isinstance(team_id, bool) for team_id in team_ids)
+ ):
+ logger.warning("error_tracking.grouping_judge.invalid_remote_config", region=region)
+ return JUDGE_OFF
+ return JudgeConfig(sample_rate=float(sample_rate), team_ids=frozenset(team_ids))
+
+
+def current_judge_config() -> JudgeConfig:
+ # Reads the payload from the locally evaluated flag definitions, so the merge activity makes no network call.
+ # A missing or disabled flag returns None, which turns the judge off.
+ try:
+ # With no definitions loaded, the SDK loads them with a blocking network call even for a
+ # local-only read, so the judge stays off until the background poller loads them.
+ if posthoganalytics.feature_flag_definitions() is None:
+ return JUDGE_OFF
+ payload = posthoganalytics.get_feature_flag_payload(
+ REMOTE_CONFIG_KEY, REMOTE_CONFIG_DISTINCT_ID, only_evaluate_locally=True, send_feature_flag_events=False
+ )
+ except Exception:
+ logger.exception("error_tracking.grouping_judge.remote_config_failed")
+ return JUDGE_OFF
+ return parse_judge_config(payload, get_instance_region())
diff --git a/products/error_tracking/backend/temporal/grouping_judge/inline.py b/products/error_tracking/backend/temporal/grouping_judge/inline.py
new file mode 100644
index 000000000000..6a8b66418a4f
--- /dev/null
+++ b/products/error_tracking/backend/temporal/grouping_judge/inline.py
@@ -0,0 +1,286 @@
+"""Judge a sample of auto-merge decisions for the metrics event. Nothing here can change a merge."""
+
+import time
+import threading
+from concurrent.futures import (
+ ThreadPoolExecutor,
+ TimeoutError as FuturesTimeoutError,
+)
+
+import structlog
+import temporalio.exceptions
+from anthropic import Anthropic
+
+from posthog.hogql import ast
+from posthog.hogql.constants import HogQLGlobalSettings
+from posthog.hogql.parser import parse_select
+from posthog.hogql.query import execute_hogql_query
+
+from posthog.clickhouse.client.connection import ClickHouseUser, Workload
+from posthog.dataclasses import frozen
+from posthog.llm.gateway_client import build_ai_gateway_anthropic_client, team_distinct_id
+from posthog.models import Team
+from posthog.temporal.common.shutdown import WorkerShuttingDownError
+from posthog.temporal.common.utils import close_db_connections
+
+from products.error_tracking.backend.temporal.fingerprint_embedding_result.judge import (
+ DEFAULT_AUTO_MERGE_CONFIG,
+ AutoMergeConfig,
+ _truncate,
+)
+from products.error_tracking.backend.temporal.fingerprint_embedding_result.types import (
+ CLOSEST_FINGERPRINTS_WINDOW,
+ AutoMergeOutcome,
+ FingerprintEmbeddingResultInputs,
+ SimilarFingerprintDistance,
+ parse_embedding_timestamp,
+)
+from products.error_tracking.backend.temporal.grouping_judge.algorithms import LIVE_ALGORITHM, decisions_for
+from products.error_tracking.backend.temporal.grouping_judge.config import current_judge_config
+from products.error_tracking.backend.temporal.grouping_judge.judge import JUDGE_MODEL, JudgeVerdict, judge_pair
+from products.error_tracking.backend.temporal.grouping_judge.sampling import (
+ JudgeSample,
+ band_for,
+ distance_bucket_for,
+ sample_pair,
+)
+from products.error_tracking.backend.temporal.grouping_judge.types import JudgeOutcome, LiveDecision
+
+logger = structlog.get_logger(__name__)
+
+AI_PRODUCT = "error_tracking_grouping_judge"
+# Bounds the ClickHouse text fetch and every judge call. A sampled issue's alert waits at most this long for them.
+TOTAL_BUDGET_SECONDS = 10.0
+TEXT_QUERY_MAX_EXECUTION_SECONDS = 3
+# A re-judge with less time than this left would most likely time out, so it is skipped.
+MIN_REJUDGE_SECONDS = 2.0
+
+_DOCUMENT_TEXT_QUERY_TEMPLATE = """
+ SELECT document_id, argMax(content, inserted_at)
+ FROM {table}
+ WHERE product = 'error_tracking'
+ AND document_type = 'fingerprint'
+ AND rendering = {{rendering}}
+ AND team_id = {{team_id}}
+ AND timestamp >= {{min_timestamp}}
+ AND document_id IN ({{document_a}}, {{document_b}})
+ GROUP BY document_id
+ """
+# Reads the same model-specific table as the closest-fingerprints query, so both texts are in it.
+DOCUMENT_TEXT_QUERY_BY_MODEL = {
+ model_name: _DOCUMENT_TEXT_QUERY_TEMPLATE.format(table=table)
+ for model_name, table in (
+ ("text-embedding-3-large-3072", "document_embeddings_text_embedding_3_large_3072"),
+ ("text-embedding-3-small-1536", "document_embeddings_text_embedding_3_small_1536"),
+ )
+}
+
+# The judge runs on its own thread so the activity can stop waiting at the budget even when a
+# call ignores its own timeout. A timed-out call keeps its thread until its own timeouts end it.
+# The ClickHouse socket has no client timeout, so a sample that finds every thread busy skips the
+# judge instead of queueing behind threads that may never come back.
+_MAX_WORKERS = 4
+_executor = ThreadPoolExecutor(max_workers=_MAX_WORKERS, thread_name_prefix="error-tracking-grouping-judge")
+_free_workers = threading.BoundedSemaphore(_MAX_WORKERS)
+
+
+@frozen
+class _JudgeResult:
+ outcome: JudgeOutcome
+ same_issue: bool | None = None
+ rejudge_same_issue: bool | None = None
+
+
+def _is_cancellation(error: BaseException) -> bool:
+ return temporalio.exceptions.is_cancelled_exception(error) or isinstance(error, WorkerShuttingDownError)
+
+
+def _live_decision(merge_outcome: AutoMergeOutcome | None) -> LiveDecision:
+ if merge_outcome is None:
+ return "error"
+ return "merge" if merge_outcome.merged_count > 0 else "no_merge"
+
+
+def _fetch_document_texts(
+ team: Team, inputs: FingerprintEmbeddingResultInputs, neighbour_fingerprint: str
+) -> dict[str, str]:
+ query_text = DOCUMENT_TEXT_QUERY_BY_MODEL.get(inputs.model_name)
+ if query_text is None:
+ raise ValueError(f"No document text query for embedding model: {inputs.model_name}")
+ # The closest-fingerprints window, so the neighbour's row is inside it.
+ min_timestamp = parse_embedding_timestamp(inputs.timestamp) - CLOSEST_FINGERPRINTS_WINDOW
+ query = parse_select(
+ query_text,
+ placeholders={
+ "rendering": ast.Constant(value=inputs.rendering),
+ "team_id": ast.Constant(value=team.id),
+ "min_timestamp": ast.Constant(value=min_timestamp),
+ "document_a": ast.Constant(value=inputs.fingerprint),
+ "document_b": ast.Constant(value=neighbour_fingerprint),
+ },
+ )
+ response = execute_hogql_query(
+ query=query,
+ team=team,
+ query_type="ErrorTrackingGroupingJudgeDocumentTexts",
+ workload=Workload.OFFLINE,
+ ch_user=ClickHouseUser.ERROR_TRACKING,
+ settings=HogQLGlobalSettings(max_execution_time=TEXT_QUERY_MAX_EXECUTION_SECONDS),
+ )
+ return {row[0]: row[1] for row in response.results if row[1]}
+
+
+def _judge_once(
+ client: Anthropic, *, new_text: str, neighbour_text: str, swapped: bool, deadline: float
+) -> JudgeVerdict:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ return JudgeVerdict(outcome="timeout")
+ first, second = (neighbour_text, new_text) if swapped else (new_text, neighbour_text)
+ return judge_pair(client, document_a=first, document_b=second, timeout_seconds=remaining)
+
+
+@close_db_connections
+def _judge_in_worker(
+ team: Team,
+ inputs: FingerprintEmbeddingResultInputs,
+ neighbour_fingerprint: str,
+ sample: JudgeSample,
+ deadline: float,
+) -> _JudgeResult:
+ # A task that waited in the queue past the deadline has no caller left to read its result.
+ if time.monotonic() >= deadline:
+ return _JudgeResult(outcome="timeout")
+ texts = _fetch_document_texts(team, inputs, neighbour_fingerprint)
+ # A workflow started before the activity input carried the text has to read it from ClickHouse.
+ new_text = inputs.content or texts.get(inputs.fingerprint)
+ neighbour_text = texts.get(neighbour_fingerprint)
+ if new_text is None or neighbour_text is None:
+ return _JudgeResult(outcome="no_text")
+ # Keeps the message and the throw site, which the renderer puts at the end of a long stack.
+ new_text, neighbour_text = _truncate(new_text), _truncate(neighbour_text)
+
+ try:
+ client = build_ai_gateway_anthropic_client(
+ ai_product=AI_PRODUCT, team_id=team.id, distinct_id=team_distinct_id(team.id)
+ )
+ except ValueError:
+ return _JudgeResult(outcome="unavailable")
+ with client:
+ verdict = _judge_once(
+ client, new_text=new_text, neighbour_text=neighbour_text, swapped=sample.swapped, deadline=deadline
+ )
+ if verdict.outcome != "judged" or not sample.rejudge or deadline - time.monotonic() < MIN_REJUDGE_SECONDS:
+ return _JudgeResult(outcome=verdict.outcome, same_issue=verdict.same_issue)
+ rejudge = _judge_once(
+ client, new_text=new_text, neighbour_text=neighbour_text, swapped=not sample.swapped, deadline=deadline
+ )
+ return _JudgeResult(outcome=verdict.outcome, same_issue=verdict.same_issue, rejudge_same_issue=rejudge.same_issue)
+
+
+def _judge(
+ team: Team, inputs: FingerprintEmbeddingResultInputs, neighbour_fingerprint: str, sample: JudgeSample
+) -> _JudgeResult:
+ deadline = time.monotonic() + TOTAL_BUDGET_SECONDS
+ try:
+ # A null approval counts as not approved.
+ if team.organization.is_ai_data_processing_approved is not True:
+ return _JudgeResult(outcome="no_consent")
+ if not _free_workers.acquire(blocking=False):
+ return _JudgeResult(outcome="unavailable")
+ try:
+ future = _executor.submit(_judge_in_worker, team, inputs, neighbour_fingerprint, sample, deadline)
+ except BaseException:
+ _free_workers.release()
+ raise
+ future.add_done_callback(lambda _: _free_workers.release())
+ return future.result(timeout=max(deadline - time.monotonic(), 0))
+ except FuturesTimeoutError:
+ return _JudgeResult(outcome="timeout")
+ except Exception as error:
+ if _is_cancellation(error):
+ # A worker drain raises into this thread while it waits. The merge has committed, and a
+ # retry cannot recover its reopened target, so the activity must still return its result.
+ return _JudgeResult(outcome="cancelled")
+ # Caught here rather than only at the entry point, so the row keeps its other judge properties.
+ logger.exception("error_tracking.grouping_judge.failed", team_id=team.id)
+ return _JudgeResult(outcome="unavailable")
+
+
+def _grouping_judge_properties(
+ team: Team,
+ inputs: FingerprintEmbeddingResultInputs,
+ closest_fingerprints: list[SimilarFingerprintDistance],
+ merge_outcome: AutoMergeOutcome | None,
+ merge_config: AutoMergeConfig,
+) -> dict[str, object]:
+ properties: dict[str, object] = {
+ "live_decision": _live_decision(merge_outcome),
+ "live_target_fingerprint": merge_outcome.target_fingerprint if merge_outcome is not None else None,
+ "live_merge_decision": merge_outcome.merge_decision if merge_outcome is not None else None,
+ # The fixed rule describes the merge only while the flag leaves every setting at its default.
+ "live_algorithm": LIVE_ALGORITHM.id if merge_config == DEFAULT_AUTO_MERGE_CONFIG else "auto_merge_config",
+ "live_auto_merge_threshold": merge_config.auto_merge_threshold,
+ "live_judge_enabled": merge_config.judge_enabled,
+ "live_judge_shadow": merge_config.judge_shadow,
+ "live_judge_cutoff": merge_config.judge_cutoff,
+ "live_judge_max_distance": merge_config.judge_max_distance,
+ "decisions": {},
+ "judge_band": None,
+ "distance_bucket": None,
+ "judge_sampled": False,
+ }
+ if not closest_fingerprints:
+ return properties
+ nearest = closest_fingerprints[0]
+ band = band_for(nearest.distance)
+ properties.update(
+ decisions=decisions_for(nearest.distance),
+ judge_band=band,
+ distance_bucket=distance_bucket_for(nearest.distance),
+ )
+ # A failed merge is retried, and the retry judges the pair, so judging it here would pay twice.
+ if merge_outcome is None:
+ return properties
+ config = current_judge_config()
+ if not config.is_enabled_for(team.id):
+ return properties
+ sample = sample_pair(team.id, inputs.fingerprint, config.sample_rate)
+ if not sample.sampled:
+ return properties
+
+ # Judge the pair the merge used, which is not rank 1 when the merge skipped it.
+ judged_fingerprint = merge_outcome.target_fingerprint or nearest.fingerprint
+ start = time.monotonic()
+ result = _judge(team, inputs, judged_fingerprint, sample)
+ # Document text and the model's reasoning hold customer data, so neither goes in the event.
+ return {
+ **properties,
+ "judge_sampled": True,
+ "judge_target_fingerprint": judged_fingerprint,
+ "judge_outcome": result.outcome,
+ "judge_same_issue": result.same_issue,
+ "judge_swapped": sample.swapped,
+ "judge_rejudge_same_issue": result.rejudge_same_issue,
+ "judge_model": JUDGE_MODEL,
+ "judge_view": inputs.rendering,
+ "judge_duration_ms": (time.monotonic() - start) * 1000,
+ }
+
+
+def grouping_judge_properties(
+ team: Team,
+ inputs: FingerprintEmbeddingResultInputs,
+ closest_fingerprints: list[SimilarFingerprintDistance],
+ merge_outcome: AutoMergeOutcome | None,
+ merge_config: AutoMergeConfig = DEFAULT_AUTO_MERGE_CONFIG,
+) -> dict[str, object]:
+ """Decision and judge properties for the metrics event. `merge_outcome` is None when the merge raised.
+
+ Never raises, and returns within about `TOTAL_BUDGET_SECONDS`.
+ """
+ try:
+ return _grouping_judge_properties(team, inputs, closest_fingerprints, merge_outcome, merge_config)
+ except Exception:
+ logger.exception("error_tracking.grouping_judge.properties_failed", team_id=team.id)
+ return {"live_decision": _live_decision(merge_outcome)}
diff --git a/products/error_tracking/backend/temporal/grouping_judge/judge.py b/products/error_tracking/backend/temporal/grouping_judge/judge.py
new file mode 100644
index 000000000000..f663f3f92d8c
--- /dev/null
+++ b/products/error_tracking/backend/temporal/grouping_judge/judge.py
@@ -0,0 +1,156 @@
+"""The LLM judge. It sees two document texts and nothing about how they were paired."""
+
+from anthropic import Anthropic, AnthropicError, APIConnectionError, APIStatusError, APITimeoutError
+from anthropic.types import ToolChoiceToolParam, ToolParam
+
+from posthog.dataclasses import frozen
+
+from products.error_tracking.backend.temporal.grouping_judge.types import JudgeOutcome
+
+JUDGE_MODEL = "claude-haiku-4-5"
+# Room for the reasoning, which comes before the verdict.
+MAX_TOKENS = 768
+# Pins the call to Bedrock in the caller's region, so EU documents never leave EU Bedrock.
+PROVIDER_HEADERS = {"X-PostHog-Provider": "bedrock"}
+
+SYSTEM_PROMPT = """\
+You are an expert in error tracking and crash triage. You will be shown two error documents, each rendered \
+as an error type + message followed by a stack trace. They come from the same application but were assigned \
+different grouping fingerprints.
+
+Decide whether they represent the SAME underlying issue — i.e. a developer fixing one would consider the \
+other a duplicate that should be grouped with it, because one fix resolves both.
+
+How to decide:
+1. Find the THROW SITE of each document: the error type, the message, and the application frames nearest to \
+where the error was raised. Frames are usually listed from the outermost caller first to the throw site last \
+(numbered frames: #00 is the throw site).
+2. If the throw sites match, the documents are the SAME issue even when they differ in:
+- dynamic values anywhere: IDs, URLs and URL path segments, numbers, percentages, timestamps, memory \
+addresses, thread/process names, session ids, user input, in any language;
+- line/column numbers, build hashes in file names, bundler module ids and chunk names, and minified frame \
+names (1–3 character identifiers such as a, Qx, b7 that change between builds);
+- the outer callers above the throw site, extra or missing frames at the bottom of the stack, stacks \
+truncated at a different depth (e.g. stack overflows), and frames inside framework or runtime internals \
+(React reconciler/scheduler, webpack runtime, unsymbolicated VM addresses);
+- a frame whose "file" is a page URL (/pricing vs /blog): it is the same inline, injected or single-page-app \
+script running on another page; compare its function names instead. When those frames have no function \
+names, they count as the same script only if the message is identical and the frame shape matches; inline \
+event handlers (onclick, ...) belong to their own page and are not shared;
+- the variable part of a templated message when the error type, template and throw site match: the requested \
+path or asset in not-found / failed-to-load errors (404 for /a vs /b, a failed-to-load error for different \
+assets), or a list of rejected fields from the same validator. When the variable part is itself an error \
+message, compare that inner error as its own throw site;
+- which chunk or module failed in chunk / dynamic-import load failures (ChunkLoadError, "error loading \
+dynamically imported module"): one issue (deploy skew or network);
+- which caller hit the same failing dependency in transient transport-level failures (no HTTP response: \
+network error, aborted request, timeout, DNS or TLS failure, connection pool exhausted, or HTTP \
+502/503/504): group by the failure, not by the caller. An HTTP 4xx or 500 response is never transient;
+- which hook or component hit a null React dispatcher ("reading 'useState'", 'useContext', ... thrown inside \
+React's hook exports): one issue (duplicate React copy or hook called outside render);
+- the page on which a browser- or extension-injected script failed (window.ethereum, __firefox__, \
+_AutofillCallbackHandler).
+3. They are DIFFERENT issues when the throw sites differ:
+- different meaningful application files, components, handlers or functions nearest the throw site (when the \
+error is raised inside framework or library code, the first application frames calling into it decide) \
+(CartSummary.tsx vs OrderHistory.tsx, component Checkout vs Profile), even with an identical message: the \
+same failure in two places needs two fixes. Named frames without a file (App, loadSettings) are application \
+frames, not minified names;
+- with no application frames at all, the library frame that calls into the framework stands in for the \
+application frame (a websocket library vs a state library, each calling into react-dom);
+- names that come from the application's own code differ: the undefined identifier (googletag vs jQuery is \
+not defined), the missing property, key or module, a nested error class in the message (NetworkError vs \
+AbortError), a different HTTP status (404 vs 401, 500 vs 503) or an HTTP status vs a network error, or a \
+different operation;
+- React minified errors (e.g. #418) with a different error code, OR different args (text vs HTML), OR \
+different route-specific files or components (settings-*.js vs billing-*.js): any one of these makes them \
+different;
+- the error type or root message differs in meaning, not just in dynamic values.
+
+Examples (SAME): "Page not found: /old-page" vs "Page not found: /docs/missing" with the same stack; "Failed \
+to load asset: /assets/icons/close.svg" vs "/assets/logo.svg"; the same recursion frames on /pricing vs \
+/blog; a connection-pool-exhausted error reached from two different request handlers; a validation error \
+listing different rejected fields from the same validator.
+Examples (DIFFERENT): "googletag is not defined" vs "jQuery is not defined"; "status code 404" vs "Network \
+Error"; the same TypeError thrown in component Checkout vs component Profile.
+
+Before deciding, name the throw site of each document. Then check the DIFFERENT rules in step 3 first: if \
+any one applies, answer DIFFERENT. Only otherwise check the SAME rules in step 2: if they explain every \
+difference, answer SAME. Product decisions in step 2 (e.g. different missing assets or 404 paths are one \
+issue) override the intuition that each needs its own fix."""
+
+USER_PROMPT_TEMPLATE = """Do these two error documents represent the same underlying issue?
+
+<document_a>
+{a}
+</document_a>
+
+<document_b>
+{b}
+</document_b>"""
+
+VERDICT_TOOL_NAME = "record_verdict"
+VERDICT_TOOL: ToolParam = {
+ "name": VERDICT_TOOL_NAME,
+ "description": "Record whether the two error documents are the same underlying issue.",
+ "input_schema": {
+ "type": "object",
+ # Reasoning first, so the model names each throw site before it commits to a verdict.
+ "properties": {
+ "reasoning": {
+ "type": "string",
+ "description": "Throw site of each document, which rule decides, in two or three short sentences",
+ },
+ "same_issue": {
+ "type": "boolean",
+ "description": "True if both documents represent the same underlying issue",
+ },
+ },
+ "required": ["reasoning", "same_issue"],
+ },
+}
+
+VERDICT_TOOL_CHOICE: ToolChoiceToolParam = {"type": "tool", "name": VERDICT_TOOL_NAME}
+
+
+@frozen
+class JudgeVerdict:
+ outcome: JudgeOutcome
+ same_issue: bool | None = None
+
+
+def _outcome_for_status(status_code: int) -> JudgeOutcome:
+ if status_code >= 500 or status_code == 429:
+ return "unavailable"
+ return "rejected"
+
+
+def judge_pair(client: Anthropic, *, document_a: str, document_b: str, timeout_seconds: float) -> JudgeVerdict:
+ try:
+ # SDK retries are off so each call stays inside the caller's time budget.
+ response = client.with_options(max_retries=0, timeout=timeout_seconds).messages.create(
+ model=JUDGE_MODEL,
+ max_tokens=MAX_TOKENS,
+ system=SYSTEM_PROMPT,
+ messages=[{"role": "user", "content": USER_PROMPT_TEMPLATE.format(a=document_a, b=document_b)}],
+ tools=[VERDICT_TOOL],
+ tool_choice=VERDICT_TOOL_CHOICE,
+ extra_headers=PROVIDER_HEADERS,
+ )
+ except APITimeoutError:
+ return JudgeVerdict(outcome="timeout")
+ except APIStatusError as error:
+ return JudgeVerdict(outcome=_outcome_for_status(error.status_code))
+ except APIConnectionError:
+ return JudgeVerdict(outcome="unavailable")
+ except AnthropicError:
+ return JudgeVerdict(outcome="invalid")
+
+ for block in response.content:
+ if block.type != "tool_use" or block.name != VERDICT_TOOL_NAME:
+ continue
+ verdict = block.input if isinstance(block.input, dict) else {}
+ same_issue = verdict.get("same_issue")
+ if isinstance(same_issue, bool):
+ return JudgeVerdict(outcome="judged", same_issue=same_issue)
+ return JudgeVerdict(outcome="invalid")
diff --git a/products/error_tracking/backend/temporal/grouping_judge/sampling.py b/products/error_tracking/backend/temporal/grouping_judge/sampling.py
new file mode 100644
index 000000000000..c98b1ccc152f
--- /dev/null
+++ b/products/error_tracking/backend/temporal/grouping_judge/sampling.py
@@ -0,0 +1,50 @@
+import hashlib
+
+from posthog.dataclasses import frozen
+
+from products.error_tracking.backend.temporal.grouping_judge.types import JudgeBand
+
+# Fixed so bands and buckets stay comparable across changes to the live auto-merge threshold. They label
+# rows for analysis and do not change the sample rate.
+LOW_BAND_EDGE = 0.019
+HIGH_BAND_EDGE = 0.08
+DISTANCE_BUCKET_EDGES: tuple[float, ...] = (0, 0.005, 0.01, 0.019, 0.03, 0.05, 0.08)
+REJUDGE_RATE = 0.1
+
+
+@frozen
+class JudgeSample:
+ sampled: bool
+ swapped: bool
+ rejudge: bool
+
+
+def band_for(distance: float) -> JudgeBand:
+ if distance < LOW_BAND_EDGE:
+ return "low"
+ if distance < HIGH_BAND_EDGE:
+ return "mid"
+ return "high"
+
+
+def distance_bucket_for(distance: float) -> str:
+ edges = DISTANCE_BUCKET_EDGES
+ for lower, upper in zip(edges, edges[1:]):
+ if distance < upper:
+ return f"{lower}-{upper}"
+ return f"{edges[-1]}+"
+
+
+def _unit_hash(purpose: str, team_id: int, fingerprint: str) -> float:
+ """A stable value in [0, 1), so an activity retry makes the same choice for the same pair."""
+ digest = hashlib.sha256(f"{purpose}:{team_id}:{fingerprint}".encode()).digest()
+ return int.from_bytes(digest[:8], "big") / 2**64
+
+
+def sample_pair(team_id: int, fingerprint: str, sample_rate: float) -> JudgeSample:
+ # Separate hashes per choice, so the swap and the re-judge are independent of being sampled.
+ return JudgeSample(
+ sampled=_unit_hash("sample", team_id, fingerprint) < sample_rate,
+ swapped=_unit_hash("swap", team_id, fingerprint) < 0.5,
+ rejudge=_unit_hash("rejudge", team_id, fingerprint) < REJUDGE_RATE,
+ )
diff --git a/products/error_tracking/backend/temporal/grouping_judge/test/test_grouping_judge.py b/products/error_tracking/backend/temporal/grouping_judge/test/test_grouping_judge.py
new file mode 100644
index 000000000000..50695b38f9a8
--- /dev/null
+++ b/products/error_tracking/backend/temporal/grouping_judge/test/test_grouping_judge.py
@@ -0,0 +1,415 @@
+import threading
+import dataclasses
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
+from typing import Any
+
+from posthog.test.base import BaseTest, ClickhouseTestMixin
+from unittest.mock import MagicMock, patch
+
+from django.test import SimpleTestCase
+
+import httpx
+from anthropic import APIConnectionError, APIStatusError, APITimeoutError
+from parameterized import parameterized
+from temporalio.exceptions import CancelledError
+
+from posthog.clickhouse.client import sync_execute
+from posthog.temporal.common.shutdown import WorkerShuttingDownError
+
+from products.error_tracking.backend.temporal.fingerprint_embedding_result.judge import (
+ DEFAULT_AUTO_MERGE_THRESHOLD,
+ AutoMergeConfig,
+)
+from products.error_tracking.backend.temporal.fingerprint_embedding_result.types import (
+ AutoMergeOutcome,
+ FingerprintEmbeddingResultInputs,
+ SimilarFingerprintDistance,
+)
+from products.error_tracking.backend.temporal.grouping_judge.algorithms import (
+ EMBEDDING_THRESHOLD_V1_DISTANCE,
+ LIVE_ALGORITHM,
+ decisions_for,
+)
+from products.error_tracking.backend.temporal.grouping_judge.config import (
+ JUDGE_OFF,
+ JudgeConfig,
+ current_judge_config,
+ parse_judge_config,
+)
+from products.error_tracking.backend.temporal.grouping_judge.inline import (
+ _fetch_document_texts,
+ grouping_judge_properties,
+)
+from products.error_tracking.backend.temporal.grouping_judge.judge import judge_pair
+from products.error_tracking.backend.temporal.grouping_judge.sampling import band_for, distance_bucket_for, sample_pair
+
+MODULE = "products.error_tracking.backend.temporal.grouping_judge"
+REQUEST = httpx.Request("POST", "http://gateway.example.com/v1/messages")
+NEW_TEXT = "TypeError: secret customer message\n at checkout (app.js)"
+NEIGHBOUR_TEXT = "TypeError: other customer message\n at checkout (app.js)"
+ALWAYS_JUDGE = JudgeConfig(sample_rate=1.0)
+NO_MERGE = AutoMergeOutcome()
+
+
+def _inputs(team_id: int = 1) -> FingerprintEmbeddingResultInputs:
+ return FingerprintEmbeddingResultInputs(
+ team_id=team_id,
+ fingerprint="fp-new",
+ rendering="type_message_and_stack",
+ timestamp="2026-10-14T09:00:00Z",
+ model_name="text-embedding-3-large-3072",
+ embedding=[0.1],
+ )
+
+
+def _tool_response(same_issue: Any, reasoning: str = "same frames") -> SimpleNamespace:
+ return SimpleNamespace(
+ content=[
+ SimpleNamespace(
+ type="tool_use", name="record_verdict", input={"same_issue": same_issue, "reasoning": reasoning}
+ )
+ ]
+ )
+
+
+def _client_returning(response: Any = None, error: Exception | None = None) -> MagicMock:
+ client = MagicMock()
+ create = client.with_options.return_value.messages.create
+ if error is not None:
+ create.side_effect = error
+ else:
+ create.return_value = response
+ return client
+
+
+class TestAlgorithms(SimpleTestCase):
+ @parameterized.expand([("below", 0.0189, "merge"), ("at_threshold", 0.019, "no_merge")])
+ def test_live_algorithm_decides_from_distance(self, _name: str, distance: float, expected: str) -> None:
+ assert decisions_for(distance) == {LIVE_ALGORITHM.id: expected}
+
+ def test_v1_matches_the_live_auto_merge_threshold(self) -> None:
+ assert EMBEDDING_THRESHOLD_V1_DISTANCE == DEFAULT_AUTO_MERGE_THRESHOLD
+
+
+class TestSampling(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("below_low_edge", 0.0189, "low", "0.01-0.019"),
+ ("at_low_edge", 0.019, "mid", "0.019-0.03"),
+ ("at_high_edge", 0.08, "high", "0.08+"),
+ ]
+ )
+ def test_band_and_bucket_edges(self, _name: str, distance: float, band: str, bucket: str) -> None:
+ assert (band_for(distance), distance_bucket_for(distance)) == (band, bucket)
+
+ def test_sampling_is_stable_across_retries(self) -> None:
+ first = [sample_pair(1, f"fp-{index}", 0.5) for index in range(200)]
+ again = [sample_pair(1, f"fp-{index}", 0.5) for index in range(200)]
+
+ assert first == again
+ assert 0 < sum(sample.sampled for sample in first) < 200
+
+
+class TestJudgeConfig(SimpleTestCase):
+ @parameterized.expand(
+ [
+ (
+ "this_region",
+ {"EU": {"sample_rate": 0.01, "team_ids": [2]}},
+ JudgeConfig(sample_rate=0.01, team_ids=frozenset({2})),
+ ),
+ ("json_string", '{"EU": {"sample_rate": 0.01}}', JudgeConfig(sample_rate=0.01)),
+ ("other_region_only", {"US": {"sample_rate": 0.01}}, JUDGE_OFF),
+ ("flag_missing_or_fetch_failed", None, JUDGE_OFF),
+ ("rate_above_one", {"EU": {"sample_rate": 5}}, JUDGE_OFF),
+ ("rate_as_string", {"EU": {"sample_rate": "0.01"}}, JUDGE_OFF),
+ ("team_ids_as_strings", {"EU": {"sample_rate": 0.01, "team_ids": ["2"]}}, JUDGE_OFF),
+ ("not_json", "{oops", JUDGE_OFF),
+ ]
+ )
+ def test_anything_but_a_valid_entry_for_this_region_is_off(
+ self, _name: str, payload: object, expected: JudgeConfig
+ ) -> None:
+ assert parse_judge_config(payload, "EU") == expected
+
+ def test_off_without_reading_the_flag_when_no_definitions_are_loaded(self) -> None:
+ with (
+ patch(f"{MODULE}.config.posthoganalytics.feature_flag_definitions", return_value=None),
+ patch(f"{MODULE}.config.posthoganalytics.get_feature_flag_payload") as get_payload,
+ ):
+ assert current_judge_config() == JUDGE_OFF
+
+ get_payload.assert_not_called()
+
+
+class TestJudgeOutcome(SimpleTestCase):
+ @parameterized.expand(
+ [
+ (
+ "server_error",
+ APIStatusError("down", response=httpx.Response(503, request=REQUEST), body=None),
+ "unavailable",
+ ),
+ (
+ "rate_limited",
+ APIStatusError("slow", response=httpx.Response(429, request=REQUEST), body=None),
+ "unavailable",
+ ),
+ ("transport", APIConnectionError(request=REQUEST), "unavailable"),
+ ("timeout", APITimeoutError(request=REQUEST), "timeout"),
+ (
+ "client_error",
+ APIStatusError("bad", response=httpx.Response(400, request=REQUEST), body=None),
+ "rejected",
+ ),
+ ]
+ )
+ def test_errors_are_not_verdicts(self, _name: str, error: Exception, outcome: str) -> None:
+ verdict = judge_pair(_client_returning(error=error), document_a="a", document_b="b", timeout_seconds=5)
+
+ assert (verdict.outcome, verdict.same_issue) == (outcome, None)
+
+ @parameterized.expand(
+ [
+ ("non_bool_answer", _tool_response("yes"), "invalid", None),
+ ("no_tool_call", SimpleNamespace(content=[SimpleNamespace(type="text", text="yes")]), "invalid", None),
+ ("tool_answer", _tool_response(False), "judged", False),
+ ]
+ )
+ def test_answer_parsing(self, _name: str, response: SimpleNamespace, outcome: str, same_issue: bool | None) -> None:
+ verdict = judge_pair(_client_returning(response), document_a="a", document_b="b", timeout_seconds=5)
+
+ assert (verdict.outcome, verdict.same_issue) == (outcome, same_issue)
+
+
+class TestGroupingJudgeProperties(SimpleTestCase):
+ def _run(
+ self,
+ client: MagicMock,
+ *,
+ approved: bool | None = True,
+ text_rows: list[tuple[str, str]] | None = None,
+ rejudge_rate: float = 0.0,
+ merge_outcome: AutoMergeOutcome | None = NO_MERGE,
+ config: JudgeConfig = ALWAYS_JUDGE,
+ input_content: str | None = None,
+ closest: list[SimilarFingerprintDistance] | None = None,
+ ) -> tuple[dict[str, object], MagicMock, MagicMock]:
+ team = MagicMock(id=1)
+ team.organization.is_ai_data_processing_approved = approved
+ rows = text_rows if text_rows is not None else [("fp-new", NEW_TEXT), ("fp-old", NEIGHBOUR_TEXT)]
+ with (
+ patch(f"{MODULE}.inline.execute_hogql_query", return_value=MagicMock(results=rows)) as hogql,
+ patch(f"{MODULE}.inline.build_ai_gateway_anthropic_client", return_value=client) as build_client,
+ patch(f"{MODULE}.sampling.REJUDGE_RATE", rejudge_rate),
+ patch(f"{MODULE}.inline.current_judge_config", return_value=config),
+ ):
+ properties = grouping_judge_properties(
+ team,
+ dataclasses.replace(_inputs(), content=input_content),
+ closest or [SimilarFingerprintDistance(fingerprint="fp-old", distance=0.01)],
+ merge_outcome,
+ )
+ return properties, hogql, build_client
+
+ @parameterized.expand(
+ [
+ ("disabled", JUDGE_OFF, NO_MERGE),
+ ("team_not_listed", JudgeConfig(sample_rate=1.0, team_ids=frozenset({2})), NO_MERGE),
+ ("merge_raised", ALWAYS_JUDGE, None),
+ ]
+ )
+ def test_off_makes_no_text_fetch_and_no_call(
+ self, _name: str, config: JudgeConfig, merge_outcome: AutoMergeOutcome | None
+ ) -> None:
+ client = _client_returning(_tool_response(True))
+
+ properties, hogql, build_client = self._run(client, merge_outcome=merge_outcome, config=config)
+
+ assert properties["judge_sampled"] is False
+ assert "judge_outcome" not in properties
+ hogql.assert_not_called()
+ build_client.assert_not_called()
+
+ @parameterized.expand([("not_approved", False), ("unset", None)])
+ def test_team_without_ai_consent_is_not_judged(self, _name: str, approved: bool | None) -> None:
+ client = _client_returning(_tool_response(True))
+
+ properties, hogql, _ = self._run(client, approved=approved)
+
+ assert properties["judge_outcome"] == "no_consent"
+ hogql.assert_not_called()
+ client.with_options.return_value.messages.create.assert_not_called()
+
+ @parameterized.expand(
+ [
+ ("neighbour_missing", [("fp-new", NEW_TEXT)], NEW_TEXT, "no_text"),
+ ("new_text_missing_and_not_in_input", [("fp-old", NEIGHBOUR_TEXT)], None, "no_text"),
+ ("new_text_not_in_clickhouse_yet", [("fp-old", NEIGHBOUR_TEXT)], NEW_TEXT, "judged"),
+ ]
+ )
+ def test_judges_only_with_both_texts(
+ self, _name: str, text_rows: list[tuple[str, str]], input_content: str | None, outcome: str
+ ) -> None:
+ client = _client_returning(_tool_response(True))
+
+ properties, _, _ = self._run(client, text_rows=text_rows, input_content=input_content)
+
+ create = client.with_options.return_value.messages.create
+ assert properties["judge_outcome"] == outcome
+ assert create.called is (outcome == "judged")
+ if create.called:
+ assert "secret customer" in create.call_args.kwargs["messages"][0]["content"]
+
+ def test_rejudge_swaps_order_and_event_carries_no_text_or_reasoning(self) -> None:
+ client = _client_returning(_tool_response(True, reasoning="REASONING-MARKER"))
+
+ properties, _, _ = self._run(
+ client,
+ rejudge_rate=1.0,
+ merge_outcome=AutoMergeOutcome(merged_count=1, target_fingerprint="fp-old"),
+ )
+
+ first_prompt, second_prompt = (
+ call.kwargs["messages"][0]["content"]
+ for call in client.with_options.return_value.messages.create.call_args_list
+ )
+ new_first_in_first_prompt = first_prompt.index("secret customer") < first_prompt.index("other customer")
+ assert new_first_in_first_prompt is not properties["judge_swapped"]
+ assert new_first_in_first_prompt != (
+ second_prompt.index("secret customer") < second_prompt.index("other customer")
+ )
+ assert {key: value for key, value in properties.items() if key != "judge_duration_ms"} == {
+ "live_decision": "merge",
+ "live_target_fingerprint": "fp-old",
+ "live_merge_decision": None,
+ "live_algorithm": "embedding_threshold_v1",
+ "live_auto_merge_threshold": 0.019,
+ "live_judge_enabled": False,
+ "live_judge_shadow": True,
+ "live_judge_cutoff": 0.9,
+ "live_judge_max_distance": 0.1,
+ "decisions": {"embedding_threshold_v1": "merge"},
+ "judge_band": "low",
+ "distance_bucket": "0.01-0.019",
+ "judge_sampled": True,
+ "judge_target_fingerprint": "fp-old",
+ "judge_outcome": "judged",
+ "judge_same_issue": True,
+ "judge_swapped": properties["judge_swapped"],
+ "judge_rejudge_same_issue": True,
+ "judge_model": "claude-haiku-4-5",
+ "judge_view": "type_message_and_stack",
+ }
+ serialized = repr(properties)
+ for leaked in ("customer message", "app.js", "REASONING-MARKER"):
+ assert leaked not in serialized
+
+ def test_judges_the_pair_the_merge_used_when_it_skipped_rank_one(self) -> None:
+ client = _client_returning(_tool_response(True))
+
+ properties, _, _ = self._run(
+ client,
+ text_rows=[("fp-new", NEW_TEXT), ("fp-old", NEIGHBOUR_TEXT), ("fp-merged", "TypeError: merged target")],
+ merge_outcome=AutoMergeOutcome(merged_count=1, target_fingerprint="fp-merged"),
+ closest=[
+ SimilarFingerprintDistance(fingerprint="fp-old", distance=0.01),
+ SimilarFingerprintDistance(fingerprint="fp-merged", distance=0.015),
+ ],
+ )
+
+ prompt = client.with_options.return_value.messages.create.call_args.kwargs["messages"][0]["content"]
+ assert properties["judge_target_fingerprint"] == "fp-merged"
+ assert "merged target" in prompt
+ assert "other customer" not in prompt
+
+ def test_long_documents_keep_their_throw_site(self) -> None:
+ client = _client_returning(_tool_response(True))
+ long_stack = "TypeError: boom\n" + " at outer_caller (app.js)\n" * 500 + " at THROW_SITE (app.js)\n"
+
+ self._run(client, input_content=long_stack)
+
+ prompt = client.with_options.return_value.messages.create.call_args.kwargs["messages"][0]["content"]
+ assert "TypeError: boom" in prompt
+ assert "THROW_SITE" in prompt
+
+ def test_records_the_merge_config_that_made_the_decision(self) -> None:
+ merge_config = AutoMergeConfig(judge_enabled=True, judge_shadow=False)
+
+ with patch(f"{MODULE}.inline.current_judge_config", return_value=JUDGE_OFF):
+ properties = grouping_judge_properties(
+ MagicMock(id=1),
+ _inputs(),
+ [SimilarFingerprintDistance(fingerprint="fp-old", distance=0.05)],
+ AutoMergeOutcome(merged_count=1, target_fingerprint="fp-old", merge_decision="judge"),
+ merge_config,
+ )
+
+ assert properties["live_algorithm"] == "auto_merge_config"
+ assert properties["live_merge_decision"] == "judge"
+ assert (properties["live_judge_enabled"], properties["live_judge_shadow"]) == (True, False)
+
+ @parameterized.expand(
+ [
+ ("temporal_cancel", CancelledError("Cancelled")),
+ ("worker_shutdown", WorkerShuttingDownError("activity-1", "merge", "queue", 1, None, None)),
+ ]
+ )
+ def test_a_drain_during_the_judge_ends_the_judge_not_the_activity(self, _name: str, error: Exception) -> None:
+ properties, _, _ = self._run(_client_returning(error=error))
+
+ assert properties["judge_outcome"] == "cancelled"
+
+ def test_skips_the_judge_when_every_worker_is_busy(self) -> None:
+ client = _client_returning(_tool_response(True))
+ no_free_workers = threading.BoundedSemaphore(1)
+ no_free_workers.acquire()
+
+ with patch(f"{MODULE}.inline._free_workers", no_free_workers):
+ properties, hogql, _ = self._run(client)
+
+ assert properties["judge_outcome"] == "unavailable"
+ hogql.assert_not_called()
+
+
+class TestFetchDocumentTexts(ClickhouseTestMixin, BaseTest):
+ def _insert(self, team_id: int, document_id: str, content: str, timestamp: datetime, inserted_at: datetime) -> None:
+ sync_execute(
+ """
+ INSERT INTO distributed_posthog_document_embeddings_text_embedding_3_large_3072 (
+ team_id, product, document_type, rendering, document_id,
+ timestamp, inserted_at, content, metadata, embedding,
+ _timestamp, _offset, _partition
+ ) VALUES
+ """,
+ [
+ (
+ team_id,
+ "error_tracking",
+ "fingerprint",
+ "type_message_and_stack",
+ document_id,
+ timestamp,
+ inserted_at,
+ content,
+ "{}",
+ [0.0] * 3072,
+ inserted_at,
+ 0,
+ 0,
+ )
+ ],
+ flush=False,
+ team_id=team_id,
+ )
+
+ def test_reads_latest_text_for_this_team_only(self) -> None:
+ created = datetime(2026, 10, 14, 9, 0, tzinfo=UTC)
+ self._insert(self.team.id, "fp-new", "old render", created, created)
+ self._insert(self.team.id, "fp-new", "x" * 7000, created, created + timedelta(minutes=1))
+ self._insert(self.team.id + 1, "fp-old", "other team text", created - timedelta(days=1), created)
+
+ texts = _fetch_document_texts(self.team, _inputs(self.team.id), "fp-old")
+
+ assert texts == {"fp-new": "x" * 7000}
diff --git a/products/error_tracking/backend/temporal/grouping_judge/types.py b/products/error_tracking/backend/temporal/grouping_judge/types.py
new file mode 100644
index 000000000000..c3e31a7c1e97
--- /dev/null
+++ b/products/error_tracking/backend/temporal/grouping_judge/types.py
@@ -0,0 +1,6 @@
+from typing import Literal
+
+Decision = Literal["merge", "no_merge"]
+LiveDecision = Literal["merge", "no_merge", "error"]
+JudgeOutcome = Literal["judged", "unavailable", "rejected", "invalid", "timeout", "cancelled", "no_text", "no_consent"]
+JudgeBand = Literal["low", "mid", "high"]
diff --git a/products/error_tracking/backend/temporal/lifecycle/issue_created/test/test_workflow.py b/products/error_tracking/backend/temporal/lifecycle/issue_created/test/test_workflow.py
index ab32e000cbbc..99198c834432 100644
--- a/products/error_tracking/backend/temporal/lifecycle/issue_created/test/test_workflow.py
+++ b/products/error_tracking/backend/temporal/lifecycle/issue_created/test/test_workflow.py
@@ -207,6 +207,27 @@ def test_embedding_failure_classification_and_capture(
assert capture_exception.called is expect_captured
+@patch("products.error_tracking.backend.temporal.lifecycle.issue_created.activities.render_stacktrace")
+@patch("products.error_tracking.backend.temporal.lifecycle.issue_created.activities.fetch_event_properties")
+@patch("products.error_tracking.backend.temporal.lifecycle.issue_created.activities.Team.objects.get")
+@patch("products.error_tracking.backend.temporal.lifecycle.issue_created.activities.generate_embedding")
+def test_embedding_hands_the_judge_its_text_before_clickhouse_has_it(
+ generate_embedding: MagicMock,
+ get_team: MagicMock,
+ fetch_event_properties: MagicMock,
+ render_stacktrace: MagicMock,
+) -> None:
+ get_team.return_value.organization.is_ai_data_processing_approved = True
+ fetch_event_properties.return_value = {"$exception_list": [{"type": "TypeError", "value": "boom"}]}
+ render_stacktrace.return_value = "TypeError: boom"
+ generate_embedding.return_value = MagicMock(embedding=[0.1, 0.2])
+
+ result = generate_issue_created_embedding_activity(_inputs("fingerprint"))
+
+ assert result.embedding is not None
+ assert result.embedding.merge_inputs.content == "TypeError: boom"
+
+
def _decision(choice: str) -> DecisionResult:
return DecisionResult(
model="severity-model",