:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(aio): slow backfills to what a rate-limited provider key allows (#129965)
Bernat Torres69a468f96f2f
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.mdindex eb2202b99938..08918b806f5f 100644--- a/docs/internal/ai-observability-judge-inputs.md+++ b/docs/internal/ai-observability-judge-inputs.md@@ -228,6 +228,9 @@ A backfill that stops on its own shows as **Interrupted**, with a stop reason.A backfill a user stops shows as **Cancelled**.Backfills read a page of units (500 by default, bounded between 1 and 1,000) and wait for every outcome before advancing progress.A backfill runs at most 16 evaluations at once (`LLMA_EVAL_BACKFILL_MAX_IN_FLIGHT`).+A run that fails because the provider stayed rate limited or unavailable through every retry halves that backfill's cap, down to 1.+After as many evaluated runs as the cap within one page, the cap rises by one, back up to the maximum.+The cap carries over between pages, but that count of evaluated runs starts again on each page, and each change logs `llma.evaluation_backfill_in_flight_limit`.A run holds one of the shared ClickHouse slots only while it reads its unit, and spends most of its time waiting on the judge.Lower the setting if backfills crowd out live evaluations.Backfill activities retry temporary database, DNS, connection, and rate-limit failures up to 5 times within 10 minutes, or within the live time limit when that is longer, with backoff from 10 seconds to one minute.diff --git a/posthog/temporal/ai_observability/evaluation_backfill.py b/posthog/temporal/ai_observability/evaluation_backfill.pyindex 1affe9654837..a6abbfc3b4ef 100644--- a/posthog/temporal/ai_observability/evaluation_backfill.py+++ b/posthog/temporal/ai_observability/evaluation_backfill.py@@ -22,7 +22,12 @@ from django.utils import timezoneimport structlogimport temporaliofrom temporalio.common import RetryPolicy, WorkflowIDReusePolicy-from temporalio.exceptions import ChildWorkflowError, WorkflowAlreadyStartedError, is_cancelled_exception+from temporalio.exceptions import (+ ApplicationError,+ ChildWorkflowError,+ WorkflowAlreadyStartedError,+ is_cancelled_exception,+)from temporalio.workflow import ParentClosePolicyfrom posthog.dataclasses import frozen@@ -30,7 +35,7 @@ from posthog.event_usage import groupsfrom posthog.models.team import Teamfrom posthog.ph_client import ph_background_capturefrom posthog.sync import database_sync_to_async-from posthog.temporal.ai_observability.evaluation_errors import USER_ERROR_SPECS+from posthog.temporal.ai_observability.evaluation_errors import USER_ERROR_SPECS, application_error_detailsfrom posthog.temporal.ai_observability.evaluation_event_io import as_utc_datetimefrom posthog.temporal.ai_observability.evaluation_types import EVALUATION_WORKFLOW_PREFIXESfrom posthog.temporal.ai_observability.evaluation_workflow_activities import RunEvaluationInputs@@ -114,6 +119,8 @@ class EvaluationBackfillInputs:# Carried across pages, so a backfill smaller than one pause cycle can still stop.failed_runs_in_a_row: int = 0pauses_in_a_row: int = 0+ # 0 starts a page at the configured maximum.+ in_flight_limit: int = 0class TickAction(StrEnum):@@ -221,6 +228,18 @@ class ChildOutcome(StrEnum):class BackfillChildOutcome:outcome: ChildOutcomestop_reason: str | None = None+ rate_limited: bool = False+++def _rate_limited(error: BaseException) -> bool:+ # The judge raises provider_unavailable once a 429 or a provider outage outlasts every retry, on+ # any key. The run workflow re-raises it, so it arrives wrapped in the child and activity errors.+ cause: BaseException | None = error+ while cause is not None:+ if isinstance(cause, ApplicationError) and cause.details:+ return application_error_details(cause).get("error_type") == "provider_unavailable"+ cause = cause.__cause__+ return Falsedef _stop_reason(result: WorkflowResult) -> str | None:@@ -281,6 +300,47 @@ class _BackfillPacing:self.pauses = 0+class _InFlightLimit:+ """Caps the runs a backfill has in flight, so a backfill slows to what the provider key allows.++ A run that failed because the provider kept rate limiting it, or stayed down, halves the cap. After as many+ evaluated runs as the cap within one page, the cap rises by one, up to the configured maximum.+ """++ def __init__(self, maximum: int, limit: int = 0) -> None:+ self.maximum = maximum+ self.limit = min(limit, maximum) if limit > 0 else maximum+ self._in_flight = 0+ self._clean_runs = 0+ self._changed = asyncio.Condition()++ async def __aenter__(self) -> None:+ async with self._changed:+ await self._changed.wait_for(lambda: self._in_flight < self.limit)+ self._in_flight += 1++ async def __aexit__(self, *_: object) -> None:+ async with self._changed:+ self._in_flight -= 1+ # Waking every queued run at a cap of 1 makes a page's scheduling quadratic.+ self._changed.notify(max(0, self.limit - self._in_flight))++ def record(self, result: BackfillChildOutcome) -> None:+ previous = self.limit+ if result.rate_limited:+ self.limit = max(1, self.limit // 2)+ self._clean_runs = 0+ elif result.outcome == ChildOutcome.EVALUATED:+ self._clean_runs += 1+ if self._clean_runs >= self.limit:+ self.limit = min(self.maximum, self.limit + 1)+ self._clean_runs = 0+ if self.limit != previous:+ temporalio.workflow.logger.info(+ "llma.evaluation_backfill_in_flight_limit", extra={"limit": self.limit, "previous": previous}+ )++def _child_outcome(result: WorkflowResult) -> ChildOutcome:# A run that returned a skip completed, so Retry remaining could not start it again. Only runs# that failed count toward the remaining units.@@ -671,7 +731,14 @@ class EvaluationBackfillWorkflow(PostHogWorkflow):async def _dispatch_bounded_batch(self, inputs: EvaluationBackfillInputs, tick: PrepareTickOutput) -> None:found = await self._find_candidates(inputs, tick)# A tick prepared by a worker that predates max_in_flight carries 0.- slots = asyncio.Semaphore(tick.max_in_flight or BACKFILL_DEFAULT_MAX_IN_FLIGHT)+ maximum = tick.max_in_flight or BACKFILL_DEFAULT_MAX_IN_FLIGHT+ # The cap changes the order runs start in, so a page that began on the fixed semaphore keeps it.+ limit = (+ _InFlightLimit(maximum, inputs.in_flight_limit)+ if temporalio.workflow.patched("evaluation-backfill-adaptive-in-flight")+ else None+ )+ slots = limit or asyncio.Semaphore(maximum)pacing = _BackfillPacing(inputs.failed_runs_in_a_row, inputs.pauses_in_a_row)started: list[str] = []@@ -681,6 +748,8 @@ class EvaluationBackfillWorkflow(PostHogWorkflow):return BackfillChildOutcome(outcome=ChildOutcome.NOT_STARTED)result = await self._run_child(inputs, tick, candidate, started)pacing.record(result)+ if limit is not None:+ limit.record(result)return resultasync def record(results: list[BackfillChildOutcome], *, cancelled: bool = False) -> AdvanceCursorOutput:@@ -731,6 +800,7 @@ class EvaluationBackfillWorkflow(PostHogWorkflow):consecutive_failures=0,failed_runs_in_a_row=pacing.failures,pauses_in_a_row=pacing.pauses,+ in_flight_limit=limit.limit if limit is not None else 0,))@@ -764,7 +834,7 @@ class EvaluationBackfillWorkflow(PostHogWorkflow):temporalio.workflow.logger.warning("llma.evaluation_backfill_child_failed", extra={"workflow_id": child.workflow_id}, exc_info=True)- return BackfillChildOutcome(outcome=ChildOutcome.FAILED)+ return BackfillChildOutcome(outcome=ChildOutcome.FAILED, rate_limited=_rate_limited(error))async def _dispatch_batch(self, inputs: EvaluationBackfillInputs, tick: PrepareTickOutput) -> bool:"""Walk one page, start a child per unit, and advance the cursor. True ends the loop."""diff --git a/posthog/temporal/ai_observability/test_evaluation_backfill.py b/posthog/temporal/ai_observability/test_evaluation_backfill.pyindex a6946596f7a3..cb14fd32b5fc 100644--- a/posthog/temporal/ai_observability/test_evaluation_backfill.py+++ b/posthog/temporal/ai_observability/test_evaluation_backfill.py@@ -424,8 +424,10 @@ class TestEvaluationBackfillWorkflow:await replayer.replay_workflow(history)@pytest.mark.asyncio- @pytest.mark.parametrize("max_in_flight,peak", [(0, 16), (4, 4)])- async def test_batch_dispatch_waits_for_outcomes_before_advancing(self, max_in_flight: int, peak: int) -> None:+ @pytest.mark.parametrize("max_in_flight,in_flight_limit,peak", [(0, 0, 16), (4, 0, 4), (4, 9, 4)])+ async def test_batch_dispatch_waits_for_outcomes_before_advancing(+ self, max_in_flight: int, in_flight_limit: int, peak: int+ ) -> None:found = dataclasses.replace(_found([_candidate(f"u{i}") for i in range(19)], exhausted=True),next_cursor_timestamp=(UNIT_TIMESTAMP - timedelta(hours=1)).isoformat(),@@ -438,7 +440,7 @@ class TestEvaluationBackfillWorkflow:})- await _run(mocks, bounded=True)+ await _run(mocks, dataclasses.replace(_inputs(), in_flight_limit=in_flight_limit), bounded=True)advances = [value for fn, value in mocks.activity_calls if fn is advance_evaluation_backfill_cursor_activity]assert mocks.peak_children == peak@@ -505,6 +507,58 @@ class TestEvaluationBackfillWorkflow:result.failed_delta,) == (4, 1, 2, 1)+ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "error_type,started_while_u1_runs,next_limit",+ [("provider_unavailable", 2, 2), ("trace_errored", 3, 3)],+ )+ async def test_a_rate_limited_run_lowers_the_in_flight_limit(+ self, error_type: str, started_while_u1_runs: int, next_limit: int+ ) -> None:+ judge_error = ActivityError(+ "judge failed",+ scheduled_event_id=1,+ started_event_id=2,+ identity="test",+ activity_type="judge",+ activity_id="1",+ retry_state=None,+ )+ judge_error.__cause__ = ApplicationError("429", {"error_type": error_type})+ run_error = ChildWorkflowError(+ "Child workflow failed",+ namespace="default",+ workflow_id="llma-hog-eval-E-u0-ingestion",+ run_id="run",+ workflow_type="run-evaluation",+ initiated_event_id=1,+ started_event_id=2,+ retry_state=None,+ )+ run_error.__cause__ = judge_error+ u1_result = asyncio.get_running_loop().create_future()+ mocks = _BackfillMocks(+ activity_results={+ prepare_evaluation_backfill_tick_activity: _tick(max_in_flight=4),+ find_evaluation_backfill_candidates_activity: _found([_candidate(f"u{i}") for i in range(3)]),+ },+ child_results_for_ids={+ "llma-hog-eval-E-u0-ingestion": run_error,+ "llma-hog-eval-E-u1-ingestion": u1_result,+ },+ )++ run = asyncio.create_task(_run(mocks, dataclasses.replace(_inputs(), in_flight_limit=2), bounded=True))+ for _ in range(50):+ await asyncio.sleep(0)+ assert len(mocks.child_calls) == started_while_u1_runs+ u1_result.set_result({"evaluation_id": "E", "evaluation_type": "hog", "skipped": False})+ continue_as_new = await run++ assert len(mocks.child_calls) == 3+ assert _advance_input(mocks).failed_delta == 1+ assert continue_as_new.call_args.args[0].in_flight_limit == next_limit+@pytest.mark.asyncioasync def test_a_cancel_mid_page_records_the_runs_already_started(self) -> None:mocks = _BackfillMocks(