:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(aio): back off emit retries and report only the final attempt (#117568)
Carlos Marchal00f82724d0c4
diff --git a/posthog/temporal/ai_observability/evaluation_workflow_activities.py b/posthog/temporal/ai_observability/evaluation_workflow_activities.pyindex 20b53e3274e7..15979d596c02 100644--- a/posthog/temporal/ai_observability/evaluation_workflow_activities.py+++ b/posthog/temporal/ai_observability/evaluation_workflow_activities.py@@ -26,7 +26,9 @@ from posthog.temporal.ai_observability.evaluation_llm_judge import DEFAULT_JUDGEfrom posthog.temporal.ai_observability.evaluation_sentiment import run_sentiment_evalfrom posthog.temporal.ai_observability.evaluation_types import EvaluationActivityResultfrom posthog.temporal.ai_observability.metrics import increment_emit_event_outcome-from posthog.temporal.ai_observability.team_capture import capture_ai_internal_for_team+from posthog.temporal.ai_observability.retries import is_last_attempt+from posthog.temporal.ai_observability.team_capture import capture_ai_internal_for_team, raise_emit_failure+from posthog.temporal.common.errors import NonReportableApplicationErrorfrom products.access_control.backend.facade.user_access_control import UserAccessControlfrom products.ai_observability.backend.models.evaluations import Evaluation, EvaluationStatus@@ -464,10 +466,10 @@ async def emit_generation_evaluation_event(inputs: EmitEvaluationEventInputs) ->logger.info("Skipping eval event emission; team over billing quota", team_id=event_data["team_id"])returnincrement_emit_event_outcome("failed")- raise- except Exception:+ raise_emit_failure(e)+ except Exception as e:increment_emit_event_outcome("failed")- raise+ raise_emit_failure(e)@temporalio.activity.defn@@ -564,8 +566,7 @@ async def run_local_evaluation_activity(inputs: RunLocalEvaluationInputs) -> Loc)except Exception as error:# Typed so the workflow counts one emit failure per run, after retries end.- raise ApplicationError(- "Failed to emit $ai_evaluation", type=EMIT_EVALUATION_EVENT_FAILED_ERROR_TYPE- ) from error+ error_class = ApplicationError if is_last_attempt() else NonReportableApplicationError+ raise error_class("Failed to emit $ai_evaluation", type=EMIT_EVALUATION_EVENT_FAILED_ERROR_TYPE) from erroremitted = Truereturn LocalEvaluationOutcome(evaluation=evaluation, result=result, emitted=emitted)diff --git a/posthog/temporal/ai_observability/run_aggregate_evaluation.py b/posthog/temporal/ai_observability/run_aggregate_evaluation.pyindex 62d593eb72ad..0a3d1002c7b2 100644--- a/posthog/temporal/ai_observability/run_aggregate_evaluation.py+++ b/posthog/temporal/ai_observability/run_aggregate_evaluation.py@@ -65,6 +65,11 @@ from posthog.temporal.ai_observability.run_trace_evaluation import (execute_trace_hog_eval_activity,execute_trace_llm_judge_activity,)+from posthog.temporal.ai_observability.team_capture import (+ EMIT_EVENT_RETRY_POLICY,+ EMIT_EVENT_SCHEDULE_TO_CLOSE_TIMEOUT,+ EMIT_EVENT_START_TO_CLOSE_TIMEOUT,+)from posthog.temporal.common.base import PostHogWorkflowfrom posthog.temporal.common.utils import close_db_connections@@ -704,8 +709,9 @@ class RunAggregateEvaluationWorkflow(PostHogWorkflow):backfill_id=inputs.backfill_id,event_timestamp=inputs.anchor_timestamp,),- schedule_to_close_timeout=timedelta(seconds=30),- retry_policy=RetryPolicy(maximum_attempts=3),+ start_to_close_timeout=EMIT_EVENT_START_TO_CLOSE_TIMEOUT,+ schedule_to_close_timeout=EMIT_EVENT_SCHEDULE_TO_CLOSE_TIMEOUT,+ retry_policy=EMIT_EVENT_RETRY_POLICY,)except Exception:increment_errors("emit_evaluation_event_failed", provider=result.get("provider"))diff --git a/posthog/temporal/ai_observability/run_evaluation.py b/posthog/temporal/ai_observability/run_evaluation.pyindex bdc9e456c375..57ed366011f7 100644--- a/posthog/temporal/ai_observability/run_evaluation.py+++ b/posthog/temporal/ai_observability/run_evaluation.py@@ -48,6 +48,11 @@ from posthog.temporal.ai_observability.evaluation_workflow_activities import (update_key_state_activity,)from posthog.temporal.ai_observability.metrics import increment_errors+from posthog.temporal.ai_observability.team_capture import (+ EMIT_EVENT_RETRY_POLICY,+ HYDRATING_EMIT_SCHEDULE_TO_CLOSE_TIMEOUT,+ HYDRATING_EMIT_START_TO_CLOSE_TIMEOUT,+)from posthog.temporal.common.base import PostHogWorkflowfrom products.ai_observability.backend.models.provider_keys import LLMProviderKey@@ -292,12 +297,15 @@ class RunEvaluationWorkflow(PostHogWorkflow):start_time=start_time,backfill_id=inputs.backfill_id,),- start_to_close_timeout=timedelta(seconds=120),+ start_to_close_timeout=HYDRATING_EMIT_START_TO_CLOSE_TIMEOUT,# Total deadline including queue wait: without it a task stuck in the queue# keeps the workflow RUNNING forever, and USE_EXISTING then blocks every later# trigger for this (evaluation, event) pair.- schedule_to_close_timeout=backfill_timeout(recover_backfill, timedelta(minutes=8)),- retry_policy=backfill_retry_policy(recover_backfill, RetryPolicy(maximum_attempts=3)),+ schedule_to_close_timeout=backfill_timeout(+ recover_backfill, HYDRATING_EMIT_SCHEDULE_TO_CLOSE_TIMEOUT+ ),+ # A hog or sentiment run is local and deterministic, so repeating it on the emit schedule is cheap.+ retry_policy=backfill_retry_policy(recover_backfill, EMIT_EVENT_RETRY_POLICY),)except temporalio.exceptions.ActivityError as e:if isinstance(e.cause, ApplicationError) and e.cause.type == EMIT_EVALUATION_EVENT_FAILED_ERROR_TYPE:@@ -385,9 +393,9 @@ class RunEvaluationWorkflow(PostHogWorkflow):# The activity reads the generation itself when it was handed a reference, and# that event can run to several MiB, so it gets the same budget as the local# evaluation activity rather than the 30s a capture call alone would need.- start_to_close_timeout=timedelta(seconds=120),- schedule_to_close_timeout=timedelta(minutes=8),- retry_policy=RetryPolicy(maximum_attempts=3),+ start_to_close_timeout=HYDRATING_EMIT_START_TO_CLOSE_TIMEOUT,+ schedule_to_close_timeout=HYDRATING_EMIT_SCHEDULE_TO_CLOSE_TIMEOUT,+ retry_policy=EMIT_EVENT_RETRY_POLICY,)except Exception:increment_errors("emit_evaluation_event_failed", provider=result.get("provider"))diff --git a/posthog/temporal/ai_observability/run_tagger.py b/posthog/temporal/ai_observability/run_tagger.pyindex ebd7d9fc30b2..3da7901899d5 100644--- a/posthog/temporal/ai_observability/run_tagger.py+++ b/posthog/temporal/ai_observability/run_tagger.py@@ -21,7 +21,13 @@ from posthog.temporal.ai_observability.evaluation_event_io import (from posthog.temporal.ai_observability.evaluation_workflow_activities import update_key_state_activityfrom posthog.temporal.ai_observability.message_utils import extract_text_from_messagesfrom posthog.temporal.ai_observability.model_resolution import ResolvedModel, model_spec-from posthog.temporal.ai_observability.team_capture import capture_ai_internal_for_team+from posthog.temporal.ai_observability.team_capture import (+ EMIT_EVENT_RETRY_POLICY,+ HYDRATING_EMIT_SCHEDULE_TO_CLOSE_TIMEOUT,+ HYDRATING_EMIT_START_TO_CLOSE_TIMEOUT,+ capture_ai_internal_for_team,+ raise_emit_failure,+)from posthog.temporal.common.base import PostHogWorkflowfrom posthog.temporal.common.utils import close_db_connections@@ -612,7 +618,9 @@ async def emit_tagger_event_activity(inputs: EmitTaggerEventInputs) -> None:if e.is_billing_limit_exceeded:logger.info("Skipping tag event emission; team over billing quota", team_id=event_data["team_id"])return- raise+ raise_emit_failure(e)+ except Exception as e:+ raise_emit_failure(e)@temporalio.activity.defn@@ -749,8 +757,9 @@ class RunTaggerWorkflow(PostHogWorkflow):result=result,start_time=start_time,),- schedule_to_close_timeout=timedelta(minutes=2),- retry_policy=RetryPolicy(maximum_attempts=3),+ start_to_close_timeout=HYDRATING_EMIT_START_TO_CLOSE_TIMEOUT,+ schedule_to_close_timeout=HYDRATING_EMIT_SCHEDULE_TO_CLOSE_TIMEOUT,+ retry_policy=EMIT_EVENT_RETRY_POLICY,)return {diff --git a/posthog/temporal/ai_observability/run_trace_evaluation.py b/posthog/temporal/ai_observability/run_trace_evaluation.pyindex 308a019794c7..735b542d5434 100644--- a/posthog/temporal/ai_observability/run_trace_evaluation.py+++ b/posthog/temporal/ai_observability/run_trace_evaluation.py@@ -70,7 +70,13 @@ from posthog.temporal.ai_observability.run_evaluation import (handle_llm_judge_activity_error,handle_terminal_user_error_result,)-from posthog.temporal.ai_observability.team_capture import capture_ai_internal_for_team+from posthog.temporal.ai_observability.team_capture import (+ EMIT_EVENT_RETRY_POLICY,+ EMIT_EVENT_SCHEDULE_TO_CLOSE_TIMEOUT,+ EMIT_EVENT_START_TO_CLOSE_TIMEOUT,+ capture_ai_internal_for_team,+ raise_emit_failure,+)from posthog.temporal.common.base import PostHogWorkflowfrom posthog.temporal.common.utils import close_db_connections@@ -841,10 +847,10 @@ async def emit_trace_evaluation_event_activity(inputs: EmitTraceEvaluationEventIlogger.info("Skipping eval event emission; team over billing quota", team_id=inputs.team_id)returnincrement_emit_event_outcome("failed")- raise- except Exception:+ raise_emit_failure(e)+ except Exception as e:increment_emit_event_outcome("failed")- raise+ raise_emit_failure(e)@temporalio.workflow.defn(name="run-trace-evaluation")@@ -940,8 +946,9 @@ class RunTraceEvaluationWorkflow(PostHogWorkflow):result=result,start_time=eval_start,),- schedule_to_close_timeout=timedelta(seconds=30),- retry_policy=RetryPolicy(maximum_attempts=3),+ start_to_close_timeout=EMIT_EVENT_START_TO_CLOSE_TIMEOUT,+ schedule_to_close_timeout=EMIT_EVENT_SCHEDULE_TO_CLOSE_TIMEOUT,+ retry_policy=EMIT_EVENT_RETRY_POLICY,)except Exception:increment_errors("emit_evaluation_event_failed", provider=result.get("provider"))diff --git a/posthog/temporal/ai_observability/team_capture.py b/posthog/temporal/ai_observability/team_capture.pyindex 7d160dccc6eb..24bc237803f9 100644--- a/posthog/temporal/ai_observability/team_capture.py+++ b/posthog/temporal/ai_observability/team_capture.py@@ -10,12 +10,16 @@ once per team per TTL.import timeimport threading-from datetime import datetime+from datetime import datetime, timedeltafrom http import HTTPStatus-from typing import Any+from typing import Any, NoReturn++from temporalio.common import RetryPolicyfrom posthog.api.capture import CaptureInternalError, capture_ai_internalfrom posthog.models.team import Team+from posthog.temporal.ai_observability.retries import is_last_attempt+from posthog.temporal.common.errors import NonReportableApplicationError# Also bounds how long emits keep using a rotated token: capture-rs accepts stale tokens at# the edge, so rotation is only picked up once this TTL lapses.@@ -25,6 +29,21 @@ _MAX_CACHED_TEAMS = 10_000_AUTH_REJECTED_STATUSES = (HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN)+# A capture-ai restart or DNS blip outlasts the urllib3 retries inside capture_ai_internal. The+# attempts wait 5s, 10s, 20s and 40s between them, so the last one starts over a minute after the first.+EMIT_EVENT_RETRY_POLICY = RetryPolicy(+ initial_interval=timedelta(seconds=5),+ backoff_coefficient=2.0,+ maximum_attempts=5,+)+# Each schedule_to_close budget fits every attempt running to its start_to_close timeout plus the+# backoff, so the attempt count ends retries and `raise_emit_failure` can tell the final attempt.+EMIT_EVENT_START_TO_CLOSE_TIMEOUT = timedelta(seconds=30)+EMIT_EVENT_SCHEDULE_TO_CLOSE_TIMEOUT = timedelta(minutes=4)+# For emits that first read their generation, which can run to several MiB.+HYDRATING_EMIT_START_TO_CLOSE_TIMEOUT = timedelta(seconds=120)+HYDRATING_EMIT_SCHEDULE_TO_CLOSE_TIMEOUT = timedelta(minutes=12)+_lock = threading.Lock()_cache: dict[int, tuple[float, str]] = {}@@ -33,6 +52,13 @@ class TeamNotFound(Exception):pass+def raise_emit_failure(error: Exception) -> NoReturn:+ """Re-raise an emit failure, kept out of error tracking while Temporal still has attempts left."""+ if is_last_attempt():+ raise error+ raise NonReportableApplicationError(str(error), type=type(error).__name__) from error++def get_team_api_token(team_id: int) -> str:"""Return the team's API token, reading Postgres at most once per TTL per worker process."""now = time.monotonic()diff --git a/posthog/temporal/ai_observability/test_team_capture.py b/posthog/temporal/ai_observability/test_team_capture.pyindex e4b68ee62003..56bedaa4aa83 100644--- a/posthog/temporal/ai_observability/test_team_capture.py+++ b/posthog/temporal/ai_observability/test_team_capture.py@@ -1,19 +1,30 @@import time-from datetime import UTC, datetime+import dataclasses+from datetime import UTC, datetime, timedelta+import time_machinefrom posthog.test.base import BaseTestfrom unittest.mock import MagicMock, patch+from django.test import SimpleTestCase+from parameterized import parameterized+from temporalio.testing import ActivityEnvironmentfrom posthog.api.capture import CaptureInternalErrorfrom posthog.models.team import Teamfrom posthog.temporal.ai_observability.team_capture import (+ EMIT_EVENT_RETRY_POLICY,+ EMIT_EVENT_SCHEDULE_TO_CLOSE_TIMEOUT,TOKEN_CACHE_TTL_SECONDS,TeamNotFound,capture_ai_internal_for_team,get_team_api_token,+ raise_emit_failure,)+from posthog.temporal.common.errors import NonReportableApplicationError++NOW = datetime(2026, 1, 1, tzinfo=UTC)def _emit(team_id: int) -> None:@@ -87,3 +98,28 @@ class TestTeamCapture(BaseTest):):with self.assertNumQueries(0):_emit(self.team.id)+++class TestRaiseEmitFailure(SimpleTestCase):+ @parameterized.expand(+ [+ ("retry_left", 1, 3600, NonReportableApplicationError),+ ("final_attempt", EMIT_EVENT_RETRY_POLICY.maximum_attempts, 3600, CaptureInternalError),+ ("backoff_lands_past_deadline", 1, 2, CaptureInternalError),+ ]+ )+ def test_only_the_final_attempt_raises_a_reportable_error(+ self, _name, attempt, seconds_to_deadline, expected_error+ ):+ schedule_to_close = EMIT_EVENT_SCHEDULE_TO_CLOSE_TIMEOUT+ env = ActivityEnvironment()+ env.info = dataclasses.replace(+ env.info,+ attempt=attempt,+ retry_policy=EMIT_EVENT_RETRY_POLICY,+ schedule_to_close_timeout=schedule_to_close,+ scheduled_time=NOW - schedule_to_close + timedelta(seconds=seconds_to_deadline),+ )+ with time_machine.travel(NOW, tick=False), self.assertRaises(Exception) as ctx:+ env.run(raise_emit_failure, CaptureInternalError("blip"))+ assert type(ctx.exception) is expected_error