:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(error-tracking): retry throttled auto-merge judge calls (#124907)
Hugues Pouillotaa8cafd076f4
diff --git a/products/error_tracking/backend/temporal/fingerprint_embedding_result/judge.py b/products/error_tracking/backend/temporal/fingerprint_embedding_result/judge.pyindex e01a0b847a10..8f9bc8d4a067 100644--- a/products/error_tracking/backend/temporal/fingerprint_embedding_result/judge.py+++ b/products/error_tracking/backend/temporal/fingerprint_embedding_result/judge.py@@ -14,6 +14,7 @@ compared against the previous one on live traffic."""import time+import randomimport dataclassesfrom typing import Literal@@ -36,6 +37,10 @@ JUDGE_MODEL: JudgeModel = JEEVES_MODELJUDGE_PROMPT_VERSION = "v5"JUDGE_QUESTION_KEY = "same_issue"MAX_DOCUMENT_CHARS = 6000 # what the judge was evaluated with+# Rate limits and overload clear within seconds; retries stay inside the judge timeout.+RETRYABLE_STATUS_CODES = frozenset({429, 529})+RETRY_BACKOFF_SECONDS = (0.25, 0.5, 1.0)+MIN_ATTEMPT_SECONDS = 1.0@dataclasses.dataclass(frozen=True)@@ -218,16 +223,28 @@ def ask_judge() -> JudgeVerdict:"""Ask the judge model for P(same issue). Raises JudgeUnavailableError when it can't answer."""start = time.monotonic()- try:- client = build_system_one_client(- model=model, ai_product="error_tracking", team_id=team_id, timeout=timeout_seconds- )- result = client.decide(- state=judge_state(document, candidate_document),- questions={JUDGE_QUESTION_KEY: NoulQuestion(instructions=JUDGE_QUESTION)},- )- except (SystemOneNotConfigured, SystemOneRequestFailed) as err:- raise JudgeUnavailableError(str(err)) from err+ remaining_seconds = timeout_seconds+ for backoff in (*RETRY_BACKOFF_SECONDS, None):+ try:+ client = build_system_one_client(+ model=model, ai_product="error_tracking", team_id=team_id, timeout=remaining_seconds+ )+ result = client.decide(+ state=judge_state(document, candidate_document),+ questions={JUDGE_QUESTION_KEY: NoulQuestion(instructions=JUDGE_QUESTION)},+ )+ break+ except SystemOneNotConfigured as err:+ raise JudgeUnavailableError(str(err)) from err+ except SystemOneRequestFailed as err:+ if backoff is None or err.status_code not in RETRYABLE_STATUS_CODES:+ raise JudgeUnavailableError(str(err)) from err+ # Jitter spreads the retries of workers that were throttled together.+ delay = backoff * random.uniform(1, 2)+ remaining_seconds = timeout_seconds - (time.monotonic() - start) - delay+ if remaining_seconds < MIN_ATTEMPT_SECONDS:+ raise JudgeUnavailableError(str(err)) from err+ time.sleep(delay)answer = result.answers.get(JUDGE_QUESTION_KEY)if not isinstance(answer, NoulAnswer):raise JudgeUnavailableError("Judge refused or gave no answer")diff --git a/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_judge.py b/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_judge.pyindex 45042905f175..b9dd521d158f 100644--- a/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_judge.py+++ b/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_judge.py@@ -126,14 +126,18 @@ class TestAutoMergeConfig:class TestAskJudge:def _ask(- self, result: SystemOneResult | Exception, model: JudgeModel | None = None+ self, result: SystemOneResult | Exception | list, model: JudgeModel | None = None) -> tuple[JudgeVerdict, MagicMock, MagicMock]:client = MagicMock()- if isinstance(result, Exception):+ if isinstance(result, Exception | list):client.decide.side_effect = resultelse:client.decide.return_value = result- with patch(f"{JUDGE}.build_system_one_client", return_value=client) as build:+ with (+ patch(f"{JUDGE}.build_system_one_client", return_value=client) as build,+ patch(f"{JUDGE}.time.sleep"),+ patch(f"{JUDGE}.random.uniform", return_value=1),+ ):if model is None:verdict = ask_judge(7, "Error: a", "Error: b", timeout_seconds=3)else:@@ -175,6 +179,47 @@ class TestAskJudge:with pytest.raises(JudgeUnavailableError):self._ask(result)+ @parameterized.expand([("rate_limited", 429), ("overloaded", 529)])+ def test_throttled_request_is_retried(self, _name: str, status_code: int) -> None:+ answer = SystemOneResult(model=JEVK5_MODEL, answers={"same_issue": NoulAnswer(probability=0.7)}, input_tokens=1)++ verdict, build, client = self._ask([SystemOneRequestFailed("throttled", status_code=status_code), answer])++ assert verdict.score == pytest.approx(0.7)+ assert client.decide.call_count == 2+ assert build.call_args_list[1].kwargs["timeout"] < 3++ @parameterized.expand([("bad_request", 400), ("transport_error", None)])+ def test_other_failures_are_not_retried(self, _name: str, status_code: int | None) -> None:+ client = MagicMock()+ client.decide.side_effect = SystemOneRequestFailed("failed", status_code=status_code)+ with (+ patch(f"{JUDGE}.build_system_one_client", return_value=client),+ patch(f"{JUDGE}.time.sleep") as sleep,+ pytest.raises(JudgeUnavailableError),+ ):+ ask_judge(7, "Error: a", "Error: b", timeout_seconds=3)+ assert client.decide.call_count == 1+ sleep.assert_not_called()++ def test_retries_stop_within_the_timeout(self) -> None:+ client = MagicMock()+ client.decide.side_effect = SystemOneRequestFailed("rate limited", status_code=429)+ clock = [100.0]++ def sleep(seconds: float) -> None:+ clock[0] += seconds++ with (+ patch(f"{JUDGE}.build_system_one_client", return_value=client),+ patch(f"{JUDGE}.time.monotonic", side_effect=lambda: clock[0]),+ patch(f"{JUDGE}.time.sleep", side_effect=sleep),+ patch(f"{JUDGE}.random.uniform", return_value=1),+ pytest.raises(JudgeUnavailableError),+ ):+ ask_judge(7, "Error: a", "Error: b", timeout_seconds=2)+ assert client.decide.call_count == 3+def test_long_documents_keep_the_message_and_the_throw_site(self) -> None:document = "TypeError: boom\n" + "outer frame\n" * 1000 + "throwSite in app.js line 1"