Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(hogql): retry jev batches when the model server is full (#119520)

Jovan Sakovicef8f1b0dac1b
diff --git a/posthog/hogql/test/test_prompt_jev.py b/posthog/hogql/test/test_prompt_jev.py
index e4494c225691..cfaabeccc5b1 100644
--- a/posthog/hogql/test/test_prompt_jev.py
+++ b/posthog/hogql/test/test_prompt_jev.py
@@ -4,7 +4,7 @@ import time
import asyncio
from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events
-from unittest.mock import patch
+from unittest.mock import AsyncMock, patch
from django.test import SimpleTestCase, override_settings
@@ -122,15 +122,41 @@ class TestPromptJev(SimpleTestCase):
PromptJevRunner(team_id=1, distinct_id=None).evaluate(PromptJevCall.parse(node), [value])
post.assert_not_called()
- @parameterized.expand([(429, "could not evaluate"), (402, "used all its AI credits")])
- def test_gateway_failure_does_not_return_a_decision(self, status: int, message: str) -> None:
+ @parameterized.expand(
+ [
+ (429, "could not evaluate", 5),
+ (529, "could not evaluate", 5),
+ (500, "could not evaluate", 1),
+ (402, "used all its AI credits", 1),
+ ]
+ )
+ def test_gateway_failure_does_not_return_a_decision(self, status: int, message: str, attempts: int) -> None:
node = parse_expr("jev('refund', 'Refund?')")
assert isinstance(node, ast.Call)
with (
- patch("httpx.AsyncClient.post", return_value=httpx.Response(status)),
+ patch("httpx.AsyncClient.post", return_value=httpx.Response(status)) as post,
+ patch("posthog.hogql.transforms.prompt_jev.asyncio.sleep", new=AsyncMock()),
self.assertRaisesRegex(QueryError, message),
):
PromptJevRunner(team_id=1, distinct_id=None).evaluate(PromptJevCall.parse(node), ["refund"])
+ self.assertEqual(post.call_count, attempts)
+
+ def test_full_model_server_is_retried_until_it_answers(self) -> None:
+ node = parse_expr("jev(body, 'Refund?')")
+ assert isinstance(node, ast.Call)
+ responses = [httpx.Response(429), httpx.Response(429)]
+
+ def full_then_answer(url: str, *, json: dict, headers: dict) -> httpx.Response:
+ return responses.pop(0) if responses else gateway_response(url, json=json, headers=headers)
+
+ with (
+ patch("httpx.AsyncClient.post", side_effect=full_then_answer) as post,
+ patch("posthog.hogql.transforms.prompt_jev.asyncio.sleep", new=AsyncMock()) as sleep,
+ ):
+ decisions = PromptJevRunner(team_id=1, distinct_id=None).evaluate(PromptJevCall.parse(node), ["refund"])
+ self.assertEqual(decisions, [0.9])
+ self.assertEqual(post.call_count, 3)
+ self.assertEqual(sleep.await_count, 2)
def test_deadline_cancels_inflight_requests_and_pending_batches(self) -> None:
node = parse_expr("jev(body, 'Refund?', batch_size := 1)")
diff --git a/posthog/hogql/transforms/prompt_jev.py b/posthog/hogql/transforms/prompt_jev.py
index d8cb96089d95..4a2215441bd5 100644
--- a/posthog/hogql/transforms/prompt_jev.py
+++ b/posthog/hogql/transforms/prompt_jev.py
@@ -2,6 +2,7 @@ import json
import math
import time
import uuid
+import random
import asyncio
from collections.abc import Callable
from dataclasses import replace
@@ -30,6 +31,7 @@ from posthog.llm.system_one import (
Question,
SystemOneNotConfigured,
SystemOneRequestFailed,
+ SystemOneResult,
)
from posthog.llm.system_one_client import GatewaySystemOneClient, build_system_one_client
@@ -43,6 +45,9 @@ logger = structlog.get_logger(__name__)
MAX_INPUT_BYTES = 8192
MAX_TOTAL_BYTES = 2 * 1024 * 1024
MAX_BATCH_BYTES = 32768
+# The model servers answer 429 when they are full and 529 when overloaded, and ask callers to back off.
+RETRYABLE_STATUS_CODES = frozenset({429, 529})
+RETRY_DELAYS_SECONDS = (0.25, 0.5, 1.0, 2.0)
class _ResultField(DatabaseField):
@@ -164,7 +169,7 @@ class PromptJevRunner:
},
)
try:
- result = await replace(client, timeout=min(remaining, 30)).adecide(state=state, questions=questions)
+ result = await self._decide_with_retries(client, state, questions)
except SystemOneRequestFailed as error:
# Log only the status, because the inputs and the gateway response body can hold customer data.
logger.warning(
@@ -186,8 +191,24 @@ class PromptJevRunner:
decisions[text] = (answer.choice, list(answer.probabilities.items()), answer.confidence)
return decisions
+ async def _decide_with_retries(
+ self, client: GatewaySystemOneClient, state: dict[str, str], questions: dict[str, Question]
+ ) -> SystemOneResult:
+ for delay in (*RETRY_DELAYS_SECONDS, None):
+ remaining = self.deadline - time.monotonic()
+ try:
+ return await replace(client, timeout=min(remaining, 30)).adecide(state=state, questions=questions)
+ except SystemOneRequestFailed as error:
+ if delay is None or error.status_code not in RETRYABLE_STATUS_CODES:
+ raise
+ wait = delay + random.uniform(0, delay)
+ if self.deadline - time.monotonic() <= wait:
+ raise
+ await asyncio.sleep(wait)
+ raise AssertionError("unreachable")
+
async def _batches(self, spec: PromptJevCall, batches: list[list[str]]) -> list[dict[str, object]]:
- semaphore = asyncio.Semaphore(4)
+ semaphore = asyncio.Semaphore(max(1, settings.HOGQL_JEV_CONCURRENCY))
async def evaluate_batch(batch: list[str]) -> dict[str, object]:
async with semaphore:
diff --git a/posthog/settings/web.py b/posthog/settings/web.py
index a9aa7834d2cb..c565b78e3749 100644
--- a/posthog/settings/web.py
+++ b/posthog/settings/web.py
@@ -1346,6 +1346,9 @@ HOGQL_PROMPT_JEV_MODEL = get_from_env("HOGQL_PROMPT_JEV_MODEL", "posthog/hogfere
# Each decision is a billed gateway call, so these bound the cost of one query.
HOGQL_JEV_MAX_ROWS = get_from_env("HOGQL_JEV_MAX_ROWS", 1000, type_cast=int)
HOGQL_JEV_MAX_DECISIONS = get_from_env("HOGQL_JEV_MAX_DECISIONS", 1000, type_cast=int)
+# Gateway requests one `jev` column keeps in flight. The model servers refuse prompt tokens past a
+# per-pod limit, so raise this only together with model capacity.
+HOGQL_JEV_CONCURRENCY = get_from_env("HOGQL_JEV_CONCURRENCY", 4, type_cast=int)
# Projected into gateway_credential.json: a JSON team_id -> tier map
# ("free"/"pro"/"enterprise") for the gateway's rate-limit bucket.