:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(hogql): retry jev batches when the model server is full (#119520)
Jovan Sakovicef8f1b0dac1b
diff --git a/posthog/hogql/test/test_prompt_jev.py b/posthog/hogql/test/test_prompt_jev.pyindex e4494c225691..cfaabeccc5b1 100644--- a/posthog/hogql/test/test_prompt_jev.py+++ b/posthog/hogql/test/test_prompt_jev.py@@ -4,7 +4,7 @@ import timeimport asynciofrom posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events-from unittest.mock import patch+from unittest.mock import AsyncMock, patchfrom django.test import SimpleTestCase, override_settings@@ -122,15 +122,41 @@ class TestPromptJev(SimpleTestCase):PromptJevRunner(team_id=1, distinct_id=None).evaluate(PromptJevCall.parse(node), [value])post.assert_not_called()- @parameterized.expand([(429, "could not evaluate"), (402, "used all its AI credits")])- def test_gateway_failure_does_not_return_a_decision(self, status: int, message: str) -> None:+ @parameterized.expand(+ [+ (429, "could not evaluate", 5),+ (529, "could not evaluate", 5),+ (500, "could not evaluate", 1),+ (402, "used all its AI credits", 1),+ ]+ )+ def test_gateway_failure_does_not_return_a_decision(self, status: int, message: str, attempts: int) -> None:node = parse_expr("jev('refund', 'Refund?')")assert isinstance(node, ast.Call)with (- patch("httpx.AsyncClient.post", return_value=httpx.Response(status)),+ patch("httpx.AsyncClient.post", return_value=httpx.Response(status)) as post,+ patch("posthog.hogql.transforms.prompt_jev.asyncio.sleep", new=AsyncMock()),self.assertRaisesRegex(QueryError, message),):PromptJevRunner(team_id=1, distinct_id=None).evaluate(PromptJevCall.parse(node), ["refund"])+ self.assertEqual(post.call_count, attempts)++ def test_full_model_server_is_retried_until_it_answers(self) -> None:+ node = parse_expr("jev(body, 'Refund?')")+ assert isinstance(node, ast.Call)+ responses = [httpx.Response(429), httpx.Response(429)]++ def full_then_answer(url: str, *, json: dict, headers: dict) -> httpx.Response:+ return responses.pop(0) if responses else gateway_response(url, json=json, headers=headers)++ with (+ patch("httpx.AsyncClient.post", side_effect=full_then_answer) as post,+ patch("posthog.hogql.transforms.prompt_jev.asyncio.sleep", new=AsyncMock()) as sleep,+ ):+ decisions = PromptJevRunner(team_id=1, distinct_id=None).evaluate(PromptJevCall.parse(node), ["refund"])+ self.assertEqual(decisions, [0.9])+ self.assertEqual(post.call_count, 3)+ self.assertEqual(sleep.await_count, 2)def test_deadline_cancels_inflight_requests_and_pending_batches(self) -> None:node = parse_expr("jev(body, 'Refund?', batch_size := 1)")diff --git a/posthog/hogql/transforms/prompt_jev.py b/posthog/hogql/transforms/prompt_jev.pyindex d8cb96089d95..4a2215441bd5 100644--- a/posthog/hogql/transforms/prompt_jev.py+++ b/posthog/hogql/transforms/prompt_jev.py@@ -2,6 +2,7 @@ import jsonimport mathimport timeimport uuid+import randomimport asynciofrom collections.abc import Callablefrom dataclasses import replace@@ -30,6 +31,7 @@ from posthog.llm.system_one import (Question,SystemOneNotConfigured,SystemOneRequestFailed,+ SystemOneResult,)from posthog.llm.system_one_client import GatewaySystemOneClient, build_system_one_client@@ -43,6 +45,9 @@ logger = structlog.get_logger(__name__)MAX_INPUT_BYTES = 8192MAX_TOTAL_BYTES = 2 * 1024 * 1024MAX_BATCH_BYTES = 32768+# The model servers answer 429 when they are full and 529 when overloaded, and ask callers to back off.+RETRYABLE_STATUS_CODES = frozenset({429, 529})+RETRY_DELAYS_SECONDS = (0.25, 0.5, 1.0, 2.0)class _ResultField(DatabaseField):@@ -164,7 +169,7 @@ class PromptJevRunner:},)try:- result = await replace(client, timeout=min(remaining, 30)).adecide(state=state, questions=questions)+ result = await self._decide_with_retries(client, state, questions)except SystemOneRequestFailed as error:# Log only the status, because the inputs and the gateway response body can hold customer data.logger.warning(@@ -186,8 +191,24 @@ class PromptJevRunner:decisions[text] = (answer.choice, list(answer.probabilities.items()), answer.confidence)return decisions+ async def _decide_with_retries(+ self, client: GatewaySystemOneClient, state: dict[str, str], questions: dict[str, Question]+ ) -> SystemOneResult:+ for delay in (*RETRY_DELAYS_SECONDS, None):+ remaining = self.deadline - time.monotonic()+ try:+ return await replace(client, timeout=min(remaining, 30)).adecide(state=state, questions=questions)+ except SystemOneRequestFailed as error:+ if delay is None or error.status_code not in RETRYABLE_STATUS_CODES:+ raise+ wait = delay + random.uniform(0, delay)+ if self.deadline - time.monotonic() <= wait:+ raise+ await asyncio.sleep(wait)+ raise AssertionError("unreachable")+async def _batches(self, spec: PromptJevCall, batches: list[list[str]]) -> list[dict[str, object]]:- semaphore = asyncio.Semaphore(4)+ semaphore = asyncio.Semaphore(max(1, settings.HOGQL_JEV_CONCURRENCY))async def evaluate_batch(batch: list[str]) -> dict[str, object]:async with semaphore:diff --git a/posthog/settings/web.py b/posthog/settings/web.pyindex a9aa7834d2cb..c565b78e3749 100644--- a/posthog/settings/web.py+++ b/posthog/settings/web.py@@ -1346,6 +1346,9 @@ HOGQL_PROMPT_JEV_MODEL = get_from_env("HOGQL_PROMPT_JEV_MODEL", "posthog/hogfere# Each decision is a billed gateway call, so these bound the cost of one query.HOGQL_JEV_MAX_ROWS = get_from_env("HOGQL_JEV_MAX_ROWS", 1000, type_cast=int)HOGQL_JEV_MAX_DECISIONS = get_from_env("HOGQL_JEV_MAX_DECISIONS", 1000, type_cast=int)+# Gateway requests one `jev` column keeps in flight. The model servers refuse prompt tokens past a+# per-pod limit, so raise this only together with model capacity.+HOGQL_JEV_CONCURRENCY = get_from_env("HOGQL_JEV_CONCURRENCY", 4, type_cast=int)# Projected into gateway_credential.json: a JSON team_id -> tier map# ("free"/"pro"/"enterprise") for the gateway's rate-limit bucket.