:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(error-tracking): select auto-merge judge via flag payload (#121303)
Hugues Pouillot40f6c3d22b20
diff --git a/products/error_tracking/backend/temporal/fingerprint_embedding_result/activities.py b/products/error_tracking/backend/temporal/fingerprint_embedding_result/activities.pyindex 026e138a2188..dbd606fd25ca 100644--- a/products/error_tracking/backend/temporal/fingerprint_embedding_result/activities.py+++ b/products/error_tracking/backend/temporal/fingerprint_embedding_result/activities.py@@ -25,7 +25,6 @@ from products.error_tracking.backend.models import ()from products.error_tracking.backend.temporal.fingerprint_embedding_result.judge import (DEFAULT_AUTO_MERGE_CONFIG,- JUDGE_MODEL,JUDGE_PROMPT_VERSION,AutoMergeConfig,JudgeUnavailableError,@@ -363,7 +362,7 @@ def _judge(remaining_seconds = deadline - time.monotonic()if remaining_seconds <= 0:raise JudgeUnavailableError("judge_timeout")- return ask_judge(team.id, inputs.content, candidate_document, remaining_seconds)+ return ask_judge(team.id, inputs.content, candidate_document, remaining_seconds, model=config.judge_model)def _judge_candidate(@@ -397,7 +396,7 @@ def _judge_candidate("judge_score": score,"judge_latency_ms": latency_ms,"judge_error": error,- "judge_model": JUDGE_MODEL,+ "judge_model": config.judge_model,"judge_prompt_version": JUDGE_PROMPT_VERSION,"judge_cutoff": config.judge_cutoff,"judge_max_distance": config.judge_max_distance,@@ -478,7 +477,7 @@ def _merge_fingerprint_into_closest_issue("merge_decision": "judge","judge_score": score,"judge_cutoff": config.judge_cutoff,- "judge_model": JUDGE_MODEL,+ "judge_model": config.judge_model,"judge_prompt_version": JUDGE_PROMPT_VERSION,}else:diff --git a/products/error_tracking/backend/temporal/fingerprint_embedding_result/judge.py b/products/error_tracking/backend/temporal/fingerprint_embedding_result/judge.pyindex f264e22dd6b8..e01a0b847a10 100644--- a/products/error_tracking/backend/temporal/fingerprint_embedding_result/judge.py+++ b/products/error_tracking/backend/temporal/fingerprint_embedding_result/judge.py@@ -1,7 +1,7 @@"""LLM judge for auto-merging fingerprints whose nearest neighbour is too far for a distance-only merge.The rule: merge automatically when the embedding distance is below `auto_merge_threshold`; between-that and `judge_max_distance`, ask Jeeves (a small classifier on the PostHog AI gateway) whether+that and `judge_max_distance`, ask the configured decision model on the PostHog AI gateway whetherthe two error documents are the same issue and merge when its probability reaches `judge_cutoff`.Every knob comes from the `error-tracking-auto-merge-config` feature flag payload, evaluated per@@ -15,6 +15,7 @@ compared against the previous one on live traffic.import timeimport dataclasses+from typing import Literalimport structlogimport posthoganalytics@@ -27,8 +28,12 @@ logger = structlog.get_logger(__name__)AUTO_MERGE_CONFIG_FLAG = "error-tracking-auto-merge-config"DEFAULT_AUTO_MERGE_THRESHOLD = 0.019-JUDGE_MODEL = "posthog/hogference/jeeves-0.1"-JUDGE_PROMPT_VERSION = "v4"+# Only approved gateway models may receive error documents; each needs its own calibrated cutoff.+type JudgeModel = Literal["posthog/hogference/jevk5-fp8-0.2", "posthog/hogference/jeeves-0.1"]+JEEVES_MODEL: JudgeModel = "posthog/hogference/jeeves-0.1"+JEVK5_MODEL: JudgeModel = "posthog/hogference/jevk5-fp8-0.2"+JUDGE_MODEL: JudgeModel = JEEVES_MODEL+JUDGE_PROMPT_VERSION = "v5"JUDGE_QUESTION_KEY = "same_issue"MAX_DOCUMENT_CHARS = 6000 # what the judge was evaluated with@@ -38,6 +43,7 @@ class AutoMergeConfig:# Merge without the judge when the nearest-neighbour distance is strictly below this.auto_merge_threshold: float = DEFAULT_AUTO_MERGE_THRESHOLDjudge_enabled: bool = False+ judge_model: JudgeModel = JUDGE_MODEL# Ask the judge for distances in [auto_merge_threshold, judge_max_distance].judge_max_distance: float = 0.1# Merge when the judge's P(same issue) is at least this.@@ -57,17 +63,32 @@ def _bounded(value: object, default: float, low: float, high: float) -> float:return float(value)+def _judge_model(value: object) -> JudgeModel | None:+ if value == JEEVES_MODEL:+ return JEEVES_MODEL+ if value == JEVK5_MODEL:+ return JEVK5_MODEL+ return None++def parse_auto_merge_config(payload: object) -> AutoMergeConfig:"""Build the config from a flag payload, keeping the default for any missing or invalid field."""payload = payload if isinstance(payload, dict) else {}raw_judge = payload.get("judge")judge: dict[str, object] = raw_judge if isinstance(raw_judge, dict) else {}defaults = AutoMergeConfig()+ judge_model = _judge_model(judge.get("model", JUDGE_MODEL))+ raw_cutoff = judge.get("cutoff")+ judge_cutoff = _bounded(raw_cutoff, defaults.judge_cutoff, 0, 1)+ explicit_cutoff_valid = _bounded(raw_cutoff, -1, 0, 1) != -1config = AutoMergeConfig(auto_merge_threshold=_bounded(payload.get("auto_merge_threshold"), defaults.auto_merge_threshold, 0, 1),- judge_enabled=judge.get("enabled") is True,+ judge_enabled=judge.get("enabled") is True+ and judge_model is not None+ and ("model" not in judge or explicit_cutoff_valid),+ judge_model=judge_model or JUDGE_MODEL,judge_max_distance=_bounded(judge.get("max_distance"), defaults.judge_max_distance, 0, 1),- judge_cutoff=_bounded(judge.get("cutoff"), defaults.judge_cutoff, 0, 1),+ judge_cutoff=judge_cutoff,judge_shadow=judge.get("shadow") is not False,judge_timeout_seconds=_bounded(judge.get("timeout_seconds"), defaults.judge_timeout_seconds, 0.1, 60),)@@ -114,14 +135,14 @@ frame names (1–3 character identifiers such as iu, o6, fT that change between- the outer callers above the throw site, extra or missing frames at the bottom of the stack, stacks \truncated at a different depth (e.g. stack overflows), and frames inside framework or runtime \internals (React reconciler/scheduler, webpack runtime, unsymbolicated VM addresses);-- a frame whose "file" is a page URL (/shop vs /games): it is the same inline, injected or \+- a frame whose "file" is a page URL (/products vs /blog): it is the same inline, injected or \single-page-app script running on another page; compare its function names instead. When those \frames have no function names, they count as the same script only if the message is identical and \the frame shape matches; inline event handlers (onclick, ...) belong to their own page and are not shared;- the variable part of a templated message when the error type, template and throw site match: the \-requested path or asset in not-found / failed-to-load errors (404 for /a vs /b, ASSET_LOAD_FAILED for \-different assets), or a list of rejected fields from the same validator. When the variable part is \-itself an error message, compare that inner error as its own throw site;+requested path or asset in not-found / failed-to-load errors (404 for /a vs /b, failed-to-load \+errors for different textures or images), or a list of rejected fields from the same validator. When \+the variable part is itself an error message, compare that inner error as its own throw site;- which chunk or module failed in chunk / dynamic-import load failures (ChunkLoadError, "error loading \dynamically imported module"): one issue (deploy skew or network);- which caller hit the same failing dependency in transient transport-level failures (no HTTP \@@ -138,19 +159,19 @@ it decide), e.g. OrderSummary.tsx vs OrderHistory.tsx, component Pricing vs Checidentical message: the same failure in two places needs two fixes. Named frames without a file \(Pricing, loadPricingPlans) are application frames, not minified names;- with no application frames at all, the library frame that calls into the framework stands in for \-the application frame (react-use-websocket vs jotai calling into react-dom);+the application frame (swr vs zustand calling into react-dom);- names that come from the application's own code differ: the undefined identifier (googletag vs \jQuery is not defined), the missing property, key or module, a nested error class in the message \(NetworkError vs AbortError), a different HTTP status (404 vs 401, 500 vs 503) or an HTTP status vs a \network error, or a different operation;- React minified errors (e.g. #418) with a different error code, OR different args (text vs HTML), OR \-different route-specific files or components (billing-*.js vs reports-*.js): any one of these makes \+different route-specific files or components (settings-*.js vs profile-*.js): any one of these makes \them different;- the error type or root message differs in meaning, not just in dynamic values.Examples (SAME): "Page not found: /config.js" vs "Page not found: /wp-json/x" with the same stack; \-"Failed to load asset at path: /Game/Items/BP_Key" vs "/Game/BP_Key"; the same recursion frames on \-/shop vs /games; a connection-pool-exhausted error reached from two different request handlers; a \+"Failed to load texture: hero.png" vs "Failed to load texture: icons/logo.png"; the same recursion \+frames on /products vs /blog; a connection-pool-exhausted error reached from two different request handlers; a \validation error listing different rejected fields from the same validator.Examples (DIFFERENT): "googletag is not defined" vs "jQuery is not defined"; "status code 404" vs \"Network Error"; the same TypeError thrown in component Pricing vs component Checkout.@@ -192,12 +213,14 @@ class JudgeVerdict:latency_ms: float-def ask_judge(team_id: int, document: str, candidate_document: str, timeout_seconds: float) -> JudgeVerdict:- """Ask Jeeves for P(same issue). Raises JudgeUnavailableError when it can't answer."""+def ask_judge(+ team_id: int, document: str, candidate_document: str, timeout_seconds: float, *, model: JudgeModel = JUDGE_MODEL+) -> JudgeVerdict:+ """Ask the judge model for P(same issue). Raises JudgeUnavailableError when it can't answer."""start = time.monotonic()try:client = build_system_one_client(- model=JUDGE_MODEL, ai_product="error_tracking", team_id=team_id, timeout=timeout_seconds+ model=model, ai_product="error_tracking", team_id=team_id, timeout=timeout_seconds)result = client.decide(state=judge_state(document, candidate_document),diff --git a/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_judge.py b/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_judge.pyindex 823c116a0b80..45042905f175 100644--- a/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_judge.py+++ b/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_judge.py@@ -24,7 +24,10 @@ from products.error_tracking.backend.temporal.fingerprint_embedding_result.activ_merge_fingerprint_into_closest_issue,)from products.error_tracking.backend.temporal.fingerprint_embedding_result.judge import (+ JEEVES_MODEL,+ JEVK5_MODEL,AutoMergeConfig,+ JudgeModel,JudgeUnavailableError,JudgeVerdict,ask_judge,@@ -61,15 +64,29 @@ class TestAutoMergeConfig:def test_no_payload_keeps_todays_rule(self) -> None:assert parse_auto_merge_config(None) == AutoMergeConfig(auto_merge_threshold=0.019, judge_enabled=False)- def test_full_payload(self) -> None:+ @parameterized.expand(+ [+ ("no_model", {}, JEEVES_MODEL),+ ("selected_model", {"model": JEVK5_MODEL}, JEVK5_MODEL),+ ]+ )+ def test_full_payload(self, _name: str, model_payload: dict[str, str], expected_model: JudgeModel) -> None:payload = {"auto_merge_threshold": 0.005,- "judge": {"enabled": True, "max_distance": 0.3, "cutoff": 0.8, "shadow": False, "timeout_seconds": 5},+ "judge": {+ "enabled": True,+ **model_payload,+ "max_distance": 0.3,+ "cutoff": 0.8,+ "shadow": False,+ "timeout_seconds": 5,+ },}assert parse_auto_merge_config(payload) == AutoMergeConfig(auto_merge_threshold=0.005,judge_enabled=True,+ judge_model=expected_model,judge_max_distance=0.3,judge_cutoff=0.8,judge_shadow=False,@@ -82,6 +99,14 @@ class TestAutoMergeConfig:("not_a_number", {"judge": {"cutoff": "0.8"}}, "judge_cutoff", 0.9),("enabled_must_be_true", {"judge": {"enabled": "yes"}}, "judge_enabled", False),("shadow_unless_false", {"judge": {"shadow": 0}}, "judge_shadow", True),+ ("unknown_model", {"judge": {"enabled": True, "model": "other", "cutoff": 0.8}}, "judge_enabled", False),+ ("model_without_cutoff", {"judge": {"enabled": True, "model": JEVK5_MODEL}}, "judge_enabled", False),+ (+ "model_with_invalid_cutoff",+ {"judge": {"enabled": True, "model": JEVK5_MODEL, "cutoff": "0.8"}},+ "judge_enabled",+ False,+ ),])def test_invalid_field_keeps_its_default(self, _name: str, payload: dict, field: str, expected: object) -> None:@@ -100,24 +125,36 @@ class TestAutoMergeConfig:class TestAskJudge:- def _ask(self, result: SystemOneResult | Exception) -> tuple[JudgeVerdict, MagicMock, MagicMock]:+ def _ask(+ self, result: SystemOneResult | Exception, model: JudgeModel | None = None+ ) -> tuple[JudgeVerdict, MagicMock, MagicMock]:client = MagicMock()if isinstance(result, Exception):client.decide.side_effect = resultelse:client.decide.return_value = resultwith patch(f"{JUDGE}.build_system_one_client", return_value=client) as build:- verdict = ask_judge(7, "Error: a", "Error: b", timeout_seconds=3)+ if model is None:+ verdict = ask_judge(7, "Error: a", "Error: b", timeout_seconds=3)+ else:+ verdict = ask_judge(7, "Error: a", "Error: b", timeout_seconds=3, model=model)return verdict, build, client- def test_returns_the_noul_probability(self) -> None:+ @parameterized.expand(+ [+ ("default", None, JEEVES_MODEL),+ ("selected", JEVK5_MODEL, JEVK5_MODEL),+ ]+ )+ def test_returns_the_noul_probability(self, _name: str, model: JudgeModel | None, expected: JudgeModel) -> None:verdict, build, client = self._ask(- SystemOneResult(model="jeeves-0.1", answers={"same_issue": NoulAnswer(probability=0.87)}, input_tokens=10)+ SystemOneResult(model=expected, answers={"same_issue": NoulAnswer(probability=0.87)}, input_tokens=10),+ model=model,)assert verdict.score == pytest.approx(0.87)assert build.call_args.kwargs == {- "model": "posthog/hogference/jeeves-0.1",+ "model": expected,"ai_product": "error_tracking","team_id": 7,"timeout": 3,@@ -126,7 +163,10 @@ class TestAskJudge:@parameterized.expand([- ("refusal", SystemOneResult(model="jeeves-0.1", answers={"same_issue": RefusalAnswer()}, input_tokens=1)),+ (+ "refusal",+ SystemOneResult(model="jevk5-fp8-0.2", answers={"same_issue": RefusalAnswer()}, input_tokens=1),+ ),("not_configured", SystemOneNotConfigured("no gateway")),("request_failed", SystemOneRequestFailed("rate limited", status_code=429)),]@@ -214,6 +254,13 @@ class TestMergeWithJudge:merged = run.events["error_tracking_issue_merged"]assert (merged["merge_decision"], merged["judge_score"]) == ("judge", 0.8)+ def test_selected_model_is_used_and_reported(self) -> None:+ run = self._merge(0.05, config=dataclasses.replace(JUDGE_ON, judge_model=JEVK5_MODEL))++ assert run.judge.call_args.kwargs["model"] == JEVK5_MODEL+ assert run.events["error_tracking_auto_merge_judged"]["judge_model"] == JEVK5_MODEL+ assert run.events["error_tracking_issue_merged"]["judge_model"] == JEVK5_MODEL+def test_judge_below_the_cutoff_does_not_merge(self) -> None:run = self._merge(0.05, score=0.79)@@ -258,6 +305,7 @@ class TestMergeWithJudge:assert run.fetch.call_args.kwargs == {"timeout_seconds": 10.0}assert run.judge.call_args.args[3] == pytest.approx(6.0)+ assert run.judge.call_args.kwargs["model"] == JUDGE_ON.judge_modeldef test_exhausted_budget_skips_the_inference(self) -> None:with patch(f"{ACTIVITIES}.time.monotonic", side_effect=[100.0, 111.0]):