Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(error-tracking): retry throttled auto-merge judge calls (#124907)

Hugues Pouillotaa8cafd076f4
diff --git a/products/error_tracking/backend/temporal/fingerprint_embedding_result/judge.py b/products/error_tracking/backend/temporal/fingerprint_embedding_result/judge.py
index e01a0b847a10..8f9bc8d4a067 100644
--- a/products/error_tracking/backend/temporal/fingerprint_embedding_result/judge.py
+++ b/products/error_tracking/backend/temporal/fingerprint_embedding_result/judge.py
@@ -14,6 +14,7 @@ compared against the previous one on live traffic.
"""
import time
+import random
import dataclasses
from typing import Literal
@@ -36,6 +37,10 @@ JUDGE_MODEL: JudgeModel = JEEVES_MODEL
JUDGE_PROMPT_VERSION = "v5"
JUDGE_QUESTION_KEY = "same_issue"
MAX_DOCUMENT_CHARS = 6000 # what the judge was evaluated with
+# Rate limits and overload clear within seconds; retries stay inside the judge timeout.
+RETRYABLE_STATUS_CODES = frozenset({429, 529})
+RETRY_BACKOFF_SECONDS = (0.25, 0.5, 1.0)
+MIN_ATTEMPT_SECONDS = 1.0
@dataclasses.dataclass(frozen=True)
@@ -218,16 +223,28 @@ def ask_judge(
) -> JudgeVerdict:
"""Ask the judge model for P(same issue). Raises JudgeUnavailableError when it can't answer."""
start = time.monotonic()
- try:
- client = build_system_one_client(
- model=model, ai_product="error_tracking", team_id=team_id, timeout=timeout_seconds
- )
- result = client.decide(
- state=judge_state(document, candidate_document),
- questions={JUDGE_QUESTION_KEY: NoulQuestion(instructions=JUDGE_QUESTION)},
- )
- except (SystemOneNotConfigured, SystemOneRequestFailed) as err:
- raise JudgeUnavailableError(str(err)) from err
+ remaining_seconds = timeout_seconds
+ for backoff in (*RETRY_BACKOFF_SECONDS, None):
+ try:
+ client = build_system_one_client(
+ model=model, ai_product="error_tracking", team_id=team_id, timeout=remaining_seconds
+ )
+ result = client.decide(
+ state=judge_state(document, candidate_document),
+ questions={JUDGE_QUESTION_KEY: NoulQuestion(instructions=JUDGE_QUESTION)},
+ )
+ break
+ except SystemOneNotConfigured as err:
+ raise JudgeUnavailableError(str(err)) from err
+ except SystemOneRequestFailed as err:
+ if backoff is None or err.status_code not in RETRYABLE_STATUS_CODES:
+ raise JudgeUnavailableError(str(err)) from err
+ # Jitter spreads the retries of workers that were throttled together.
+ delay = backoff * random.uniform(1, 2)
+ remaining_seconds = timeout_seconds - (time.monotonic() - start) - delay
+ if remaining_seconds < MIN_ATTEMPT_SECONDS:
+ raise JudgeUnavailableError(str(err)) from err
+ time.sleep(delay)
answer = result.answers.get(JUDGE_QUESTION_KEY)
if not isinstance(answer, NoulAnswer):
raise JudgeUnavailableError("Judge refused or gave no answer")
diff --git a/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_judge.py b/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_judge.py
index 45042905f175..b9dd521d158f 100644
--- a/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_judge.py
+++ b/products/error_tracking/backend/temporal/fingerprint_embedding_result/test/test_judge.py
@@ -126,14 +126,18 @@ class TestAutoMergeConfig:
class TestAskJudge:
def _ask(
- self, result: SystemOneResult | Exception, model: JudgeModel | None = None
+ self, result: SystemOneResult | Exception | list, model: JudgeModel | None = None
) -> tuple[JudgeVerdict, MagicMock, MagicMock]:
client = MagicMock()
- if isinstance(result, Exception):
+ if isinstance(result, Exception | list):
client.decide.side_effect = result
else:
client.decide.return_value = result
- with patch(f"{JUDGE}.build_system_one_client", return_value=client) as build:
+ with (
+ patch(f"{JUDGE}.build_system_one_client", return_value=client) as build,
+ patch(f"{JUDGE}.time.sleep"),
+ patch(f"{JUDGE}.random.uniform", return_value=1),
+ ):
if model is None:
verdict = ask_judge(7, "Error: a", "Error: b", timeout_seconds=3)
else:
@@ -175,6 +179,47 @@ class TestAskJudge:
with pytest.raises(JudgeUnavailableError):
self._ask(result)
+ @parameterized.expand([("rate_limited", 429), ("overloaded", 529)])
+ def test_throttled_request_is_retried(self, _name: str, status_code: int) -> None:
+ answer = SystemOneResult(model=JEVK5_MODEL, answers={"same_issue": NoulAnswer(probability=0.7)}, input_tokens=1)
+
+ verdict, build, client = self._ask([SystemOneRequestFailed("throttled", status_code=status_code), answer])
+
+ assert verdict.score == pytest.approx(0.7)
+ assert client.decide.call_count == 2
+ assert build.call_args_list[1].kwargs["timeout"] < 3
+
+ @parameterized.expand([("bad_request", 400), ("transport_error", None)])
+ def test_other_failures_are_not_retried(self, _name: str, status_code: int | None) -> None:
+ client = MagicMock()
+ client.decide.side_effect = SystemOneRequestFailed("failed", status_code=status_code)
+ with (
+ patch(f"{JUDGE}.build_system_one_client", return_value=client),
+ patch(f"{JUDGE}.time.sleep") as sleep,
+ pytest.raises(JudgeUnavailableError),
+ ):
+ ask_judge(7, "Error: a", "Error: b", timeout_seconds=3)
+ assert client.decide.call_count == 1
+ sleep.assert_not_called()
+
+ def test_retries_stop_within_the_timeout(self) -> None:
+ client = MagicMock()
+ client.decide.side_effect = SystemOneRequestFailed("rate limited", status_code=429)
+ clock = [100.0]
+
+ def sleep(seconds: float) -> None:
+ clock[0] += seconds
+
+ with (
+ patch(f"{JUDGE}.build_system_one_client", return_value=client),
+ patch(f"{JUDGE}.time.monotonic", side_effect=lambda: clock[0]),
+ patch(f"{JUDGE}.time.sleep", side_effect=sleep),
+ patch(f"{JUDGE}.random.uniform", return_value=1),
+ pytest.raises(JudgeUnavailableError),
+ ):
+ ask_judge(7, "Error: a", "Error: b", timeout_seconds=2)
+ assert client.decide.call_count == 3
+
def test_long_documents_keep_the_message_and_the_throw_site(self) -> None:
document = "TypeError: boom\n" + "outer frame\n" * 1000 + "throwSite in app.js line 1"