Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(error-tracking): select auto-merge judge via flag payload (#121303)

Hugues Pouillot40f6c3d22b20
diff --git a/products/error_tracking/backend/temporal/fingerprint_embedding_result/activities.py b/products/error_tracking/backend/temporal/fingerprint_embedding_result/activities.py
index 026e138a2188..dbd606fd25ca 100644
--- a/products/error_tracking/backend/temporal/fingerprint_embedding_result/activities.py
+++ b/products/error_tracking/backend/temporal/fingerprint_embedding_result/activities.py
@@ -25,7 +25,6 @@ from products.error_tracking.backend.models import (
)
from products.error_tracking.backend.temporal.fingerprint_embedding_result.judge import (
DEFAULT_AUTO_MERGE_CONFIG,
- JUDGE_MODEL,
JUDGE_PROMPT_VERSION,
AutoMergeConfig,
JudgeUnavailableError,
@@ -363,7 +362,7 @@ def _judge(
remaining_seconds = deadline - time.monotonic()
if remaining_seconds <= 0:
raise JudgeUnavailableError("judge_timeout")
- return ask_judge(team.id, inputs.content, candidate_document, remaining_seconds)
+ return ask_judge(team.id, inputs.content, candidate_document, remaining_seconds, model=config.judge_model)
def _judge_candidate(
@@ -397,7 +396,7 @@ def _judge_candidate(
"judge_score": score,
"judge_latency_ms": latency_ms,
"judge_error": error,
- "judge_model": JUDGE_MODEL,
+ "judge_model": config.judge_model,
"judge_prompt_version": JUDGE_PROMPT_VERSION,
"judge_cutoff": config.judge_cutoff,
"judge_max_distance": config.judge_max_distance,
@@ -478,7 +477,7 @@ def _merge_fingerprint_into_closest_issue(
"merge_decision": "judge",
"judge_score": score,
"judge_cutoff": config.judge_cutoff,
- "judge_model": JUDGE_MODEL,
+ "judge_model": config.judge_model,
"judge_prompt_version": JUDGE_PROMPT_VERSION,
}
else:
diff --git a/products/error_tracking/backend/temporal/fingerprint_embedding_result/judge.py b/products/error_tracking/backend/temporal/fingerprint_embedding_result/judge.py
index f264e22dd6b8..e01a0b847a10 100644
--- a/products/error_tracking/backend/temporal/fingerprint_embedding_result/judge.py
+++ b/products/error_tracking/backend/temporal/fingerprint_embedding_result/judge.py
@@ -1,7 +1,7 @@
"""LLM judge for auto-merging fingerprints whose nearest neighbour is too far for a distance-only merge.
The rule: merge automatically when the embedding distance is below `auto_merge_threshold`; between
-that and `judge_max_distance`, ask Jeeves (a small classifier on the PostHog AI gateway) whether
+that and `judge_max_distance`, ask the configured decision model on the PostHog AI gateway whether
the two error documents are the same issue and merge when its probability reaches `judge_cutoff`.
Every knob comes from the `error-tracking-auto-merge-config` feature flag payload, evaluated per
@@ -15,6 +15,7 @@ compared against the previous one on live traffic.
import time
import dataclasses
+from typing import Literal
import structlog
import posthoganalytics
@@ -27,8 +28,12 @@ logger = structlog.get_logger(__name__)
AUTO_MERGE_CONFIG_FLAG = "error-tracking-auto-merge-config"
DEFAULT_AUTO_MERGE_THRESHOLD = 0.019
-JUDGE_MODEL = "posthog/hogference/jeeves-0.1"
-JUDGE_PROMPT_VERSION = "v4"
+# Only approved gateway models may receive error documents; each needs its own calibrated cutoff.
+type JudgeModel = Literal["posthog/hogference/jevk5-fp8-0.2", "posthog/hogference/jeeves-0.1"]
+JEEVES_MODEL: JudgeModel = "posthog/hogference/jeeves-0.1"
+JEVK5_MODEL: JudgeModel = "posthog/hogference/jevk5-fp8-0.2"
+JUDGE_MODEL: JudgeModel = JEEVES_MODEL
+JUDGE_PROMPT_VERSION = "v5"
JUDGE_QUESTION_KEY = "same_issue"
MAX_DOCUMENT_CHARS = 6000 # what the judge was evaluated with
@@ -38,6 +43,7 @@ class AutoMergeConfig:
# Merge without the judge when the nearest-neighbour distance is strictly below this.
auto_merge_threshold: float = DEFAULT_AUTO_MERGE_THRESHOLD
judge_enabled: bool = False
+ judge_model: JudgeModel = JUDGE_MODEL
# Ask the judge for distances in [auto_merge_threshold, judge_max_distance].
judge_max_distance: float = 0.1
# Merge when the judge's P(same issue) is at least this.
@@ -57,17 +63,32 @@ def _bounded(value: object, default: float, low: float, high: float) -> float:
return float(value)
+def _judge_model(value: object) -> JudgeModel | None:
+ if value == JEEVES_MODEL:
+ return JEEVES_MODEL
+ if value == JEVK5_MODEL:
+ return JEVK5_MODEL
+ return None
+
+
def parse_auto_merge_config(payload: object) -> AutoMergeConfig:
"""Build the config from a flag payload, keeping the default for any missing or invalid field."""
payload = payload if isinstance(payload, dict) else {}
raw_judge = payload.get("judge")
judge: dict[str, object] = raw_judge if isinstance(raw_judge, dict) else {}
defaults = AutoMergeConfig()
+ judge_model = _judge_model(judge.get("model", JUDGE_MODEL))
+ raw_cutoff = judge.get("cutoff")
+ judge_cutoff = _bounded(raw_cutoff, defaults.judge_cutoff, 0, 1)
+ explicit_cutoff_valid = _bounded(raw_cutoff, -1, 0, 1) != -1
config = AutoMergeConfig(
auto_merge_threshold=_bounded(payload.get("auto_merge_threshold"), defaults.auto_merge_threshold, 0, 1),
- judge_enabled=judge.get("enabled") is True,
+ judge_enabled=judge.get("enabled") is True
+ and judge_model is not None
+ and ("model" not in judge or explicit_cutoff_valid),
+ judge_model=judge_model or JUDGE_MODEL,
judge_max_distance=_bounded(judge.get("max_distance"), defaults.judge_max_distance, 0, 1),
- judge_cutoff=_bounded(judge.get("cutoff"), defaults.judge_cutoff, 0, 1),
+ judge_cutoff=judge_cutoff,
judge_shadow=judge.get("shadow") is not False,
judge_timeout_seconds=_bounded(judge.get("timeout_seconds"), defaults.judge_timeout_seconds, 0.1, 60),
)
@@ -114,14 +135,14 @@ frame names (1–3 character identifiers such as iu, o6, fT that change between
- the outer callers above the throw site, extra or missing frames at the bottom of the stack, stacks \
truncated at a different depth (e.g. stack overflows), and frames inside framework or runtime \
internals (React reconciler/scheduler, webpack runtime, unsymbolicated VM addresses);
-- a frame whose "file" is a page URL (/shop vs /games): it is the same inline, injected or \
+- a frame whose "file" is a page URL (/products vs /blog): it is the same inline, injected or \
single-page-app script running on another page; compare its function names instead. When those \
frames have no function names, they count as the same script only if the message is identical and \
the frame shape matches; inline event handlers (onclick, ...) belong to their own page and are not shared;
- the variable part of a templated message when the error type, template and throw site match: the \
-requested path or asset in not-found / failed-to-load errors (404 for /a vs /b, ASSET_LOAD_FAILED for \
-different assets), or a list of rejected fields from the same validator. When the variable part is \
-itself an error message, compare that inner error as its own throw site;
+requested path or asset in not-found / failed-to-load errors (404 for /a vs /b, failed-to-load \
+errors for different textures or images), or a list of rejected fields from the same validator. When \
+the variable part is itself an error message, compare that inner error as its own throw site;
- which chunk or module failed in chunk / dynamic-import load failures (ChunkLoadError, "error loading \
dynamically imported module"): one issue (deploy skew or network);
- which caller hit the same failing dependency in transient transport-level failures (no HTTP \
@@ -138,19 +159,19 @@ it decide), e.g. OrderSummary.tsx vs OrderHistory.tsx, component Pricing vs Chec
identical message: the same failure in two places needs two fixes. Named frames without a file \
(Pricing, loadPricingPlans) are application frames, not minified names;
- with no application frames at all, the library frame that calls into the framework stands in for \
-the application frame (react-use-websocket vs jotai calling into react-dom);
+the application frame (swr vs zustand calling into react-dom);
- names that come from the application's own code differ: the undefined identifier (googletag vs \
jQuery is not defined), the missing property, key or module, a nested error class in the message \
(NetworkError vs AbortError), a different HTTP status (404 vs 401, 500 vs 503) or an HTTP status vs a \
network error, or a different operation;
- React minified errors (e.g. #418) with a different error code, OR different args (text vs HTML), OR \
-different route-specific files or components (billing-*.js vs reports-*.js): any one of these makes \
+different route-specific files or components (settings-*.js vs profile-*.js): any one of these makes \
them different;
- the error type or root message differs in meaning, not just in dynamic values.
Examples (SAME): "Page not found: /config.js" vs "Page not found: /wp-json/x" with the same stack; \
-"Failed to load asset at path: /Game/Items/BP_Key" vs "/Game/BP_Key"; the same recursion frames on \
-/shop vs /games; a connection-pool-exhausted error reached from two different request handlers; a \
+"Failed to load texture: hero.png" vs "Failed to load texture: icons/logo.png"; the same recursion \
+frames on /products vs /blog; a connection-pool-exhausted error reached from two different request handlers; a \
validation error listing different rejected fields from the same validator.
Examples (DIFFERENT): "googletag is not defined" vs "jQuery is not defined"; "status code 404" vs \
"Network Error"; the same TypeError thrown in component Pricing vs component Checkout.
@@ -192,12 +213,14 @@ class JudgeVerdict:
latency_ms: float
-def ask_judge(team_id: int, document: str, candidate_document: str, timeout_seconds: float) -> JudgeVerdict:
- """Ask Jeeves for P(same issue). Raises JudgeUnavailableError when it can't answer."""
+def ask_judge(
+ team_id: int, document: str, candidate_document: str, timeout_seconds: float, *, model: JudgeModel = JUDGE_MODEL
+) -> JudgeVerdict:
+ """Ask the judge model for P(same issue). Raises JudgeUnavailableError when it can't answer."""
start = time.monotonic()
try:
client = build_system_one_client(
- model=JUDGE_MODEL, ai_product="error_tracking", team_id=team_id, timeout=timeout_seconds
+ model=model, ai_product="error_tracking", team_id=team_id, timeout=timeout_seconds
)
result = client.decide(
state=judge_state(document, candidate_document),
diff --git a/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_judge.py b/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_judge.py
index 823c116a0b80..45042905f175 100644
--- a/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_judge.py
+++ b/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_judge.py
@@ -24,7 +24,10 @@ from products.error_tracking.backend.temporal.fingerprint_embedding_result.activ
_merge_fingerprint_into_closest_issue,
)
from products.error_tracking.backend.temporal.fingerprint_embedding_result.judge import (
+ JEEVES_MODEL,
+ JEVK5_MODEL,
AutoMergeConfig,
+ JudgeModel,
JudgeUnavailableError,
JudgeVerdict,
ask_judge,
@@ -61,15 +64,29 @@ class TestAutoMergeConfig:
def test_no_payload_keeps_todays_rule(self) -> None:
assert parse_auto_merge_config(None) == AutoMergeConfig(auto_merge_threshold=0.019, judge_enabled=False)
- def test_full_payload(self) -> None:
+ @parameterized.expand(
+ [
+ ("no_model", {}, JEEVES_MODEL),
+ ("selected_model", {"model": JEVK5_MODEL}, JEVK5_MODEL),
+ ]
+ )
+ def test_full_payload(self, _name: str, model_payload: dict[str, str], expected_model: JudgeModel) -> None:
payload = {
"auto_merge_threshold": 0.005,
- "judge": {"enabled": True, "max_distance": 0.3, "cutoff": 0.8, "shadow": False, "timeout_seconds": 5},
+ "judge": {
+ "enabled": True,
+ **model_payload,
+ "max_distance": 0.3,
+ "cutoff": 0.8,
+ "shadow": False,
+ "timeout_seconds": 5,
+ },
}
assert parse_auto_merge_config(payload) == AutoMergeConfig(
auto_merge_threshold=0.005,
judge_enabled=True,
+ judge_model=expected_model,
judge_max_distance=0.3,
judge_cutoff=0.8,
judge_shadow=False,
@@ -82,6 +99,14 @@ class TestAutoMergeConfig:
("not_a_number", {"judge": {"cutoff": "0.8"}}, "judge_cutoff", 0.9),
("enabled_must_be_true", {"judge": {"enabled": "yes"}}, "judge_enabled", False),
("shadow_unless_false", {"judge": {"shadow": 0}}, "judge_shadow", True),
+ ("unknown_model", {"judge": {"enabled": True, "model": "other", "cutoff": 0.8}}, "judge_enabled", False),
+ ("model_without_cutoff", {"judge": {"enabled": True, "model": JEVK5_MODEL}}, "judge_enabled", False),
+ (
+ "model_with_invalid_cutoff",
+ {"judge": {"enabled": True, "model": JEVK5_MODEL, "cutoff": "0.8"}},
+ "judge_enabled",
+ False,
+ ),
]
)
def test_invalid_field_keeps_its_default(self, _name: str, payload: dict, field: str, expected: object) -> None:
@@ -100,24 +125,36 @@ class TestAutoMergeConfig:
class TestAskJudge:
- def _ask(self, result: SystemOneResult | Exception) -> tuple[JudgeVerdict, MagicMock, MagicMock]:
+ def _ask(
+ self, result: SystemOneResult | Exception, model: JudgeModel | None = None
+ ) -> tuple[JudgeVerdict, MagicMock, MagicMock]:
client = MagicMock()
if isinstance(result, Exception):
client.decide.side_effect = result
else:
client.decide.return_value = result
with patch(f"{JUDGE}.build_system_one_client", return_value=client) as build:
- verdict = ask_judge(7, "Error: a", "Error: b", timeout_seconds=3)
+ if model is None:
+ verdict = ask_judge(7, "Error: a", "Error: b", timeout_seconds=3)
+ else:
+ verdict = ask_judge(7, "Error: a", "Error: b", timeout_seconds=3, model=model)
return verdict, build, client
- def test_returns_the_noul_probability(self) -> None:
+ @parameterized.expand(
+ [
+ ("default", None, JEEVES_MODEL),
+ ("selected", JEVK5_MODEL, JEVK5_MODEL),
+ ]
+ )
+ def test_returns_the_noul_probability(self, _name: str, model: JudgeModel | None, expected: JudgeModel) -> None:
verdict, build, client = self._ask(
- SystemOneResult(model="jeeves-0.1", answers={"same_issue": NoulAnswer(probability=0.87)}, input_tokens=10)
+ SystemOneResult(model=expected, answers={"same_issue": NoulAnswer(probability=0.87)}, input_tokens=10),
+ model=model,
)
assert verdict.score == pytest.approx(0.87)
assert build.call_args.kwargs == {
- "model": "posthog/hogference/jeeves-0.1",
+ "model": expected,
"ai_product": "error_tracking",
"team_id": 7,
"timeout": 3,
@@ -126,7 +163,10 @@ class TestAskJudge:
@parameterized.expand(
[
- ("refusal", SystemOneResult(model="jeeves-0.1", answers={"same_issue": RefusalAnswer()}, input_tokens=1)),
+ (
+ "refusal",
+ SystemOneResult(model="jevk5-fp8-0.2", answers={"same_issue": RefusalAnswer()}, input_tokens=1),
+ ),
("not_configured", SystemOneNotConfigured("no gateway")),
("request_failed", SystemOneRequestFailed("rate limited", status_code=429)),
]
@@ -214,6 +254,13 @@ class TestMergeWithJudge:
merged = run.events["error_tracking_issue_merged"]
assert (merged["merge_decision"], merged["judge_score"]) == ("judge", 0.8)
+ def test_selected_model_is_used_and_reported(self) -> None:
+ run = self._merge(0.05, config=dataclasses.replace(JUDGE_ON, judge_model=JEVK5_MODEL))
+
+ assert run.judge.call_args.kwargs["model"] == JEVK5_MODEL
+ assert run.events["error_tracking_auto_merge_judged"]["judge_model"] == JEVK5_MODEL
+ assert run.events["error_tracking_issue_merged"]["judge_model"] == JEVK5_MODEL
+
def test_judge_below_the_cutoff_does_not_merge(self) -> None:
run = self._merge(0.05, score=0.79)
@@ -258,6 +305,7 @@ class TestMergeWithJudge:
assert run.fetch.call_args.kwargs == {"timeout_seconds": 10.0}
assert run.judge.call_args.args[3] == pytest.approx(6.0)
+ assert run.judge.call_args.kwargs["model"] == JUDGE_ON.judge_model
def test_exhausted_budget_skips_the_inference(self) -> None:
with patch(f"{ACTIVITIES}.time.monotonic", side_effect=[100.0, 111.0]):