Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(aio): back off emit retries and report only the final attempt (#117568)

Carlos Marchal00f82724d0c4
diff --git a/posthog/temporal/ai_observability/evaluation_workflow_activities.py b/posthog/temporal/ai_observability/evaluation_workflow_activities.py
index 20b53e3274e7..15979d596c02 100644
--- a/posthog/temporal/ai_observability/evaluation_workflow_activities.py
+++ b/posthog/temporal/ai_observability/evaluation_workflow_activities.py
@@ -26,7 +26,9 @@ from posthog.temporal.ai_observability.evaluation_llm_judge import DEFAULT_JUDGE
from posthog.temporal.ai_observability.evaluation_sentiment import run_sentiment_eval
from posthog.temporal.ai_observability.evaluation_types import EvaluationActivityResult
from posthog.temporal.ai_observability.metrics import increment_emit_event_outcome
-from posthog.temporal.ai_observability.team_capture import capture_ai_internal_for_team
+from posthog.temporal.ai_observability.retries import is_last_attempt
+from posthog.temporal.ai_observability.team_capture import capture_ai_internal_for_team, raise_emit_failure
+from posthog.temporal.common.errors import NonReportableApplicationError
from products.access_control.backend.facade.user_access_control import UserAccessControl
from products.ai_observability.backend.models.evaluations import Evaluation, EvaluationStatus
@@ -464,10 +466,10 @@ async def emit_generation_evaluation_event(inputs: EmitEvaluationEventInputs) ->
logger.info("Skipping eval event emission; team over billing quota", team_id=event_data["team_id"])
return
increment_emit_event_outcome("failed")
- raise
- except Exception:
+ raise_emit_failure(e)
+ except Exception as e:
increment_emit_event_outcome("failed")
- raise
+ raise_emit_failure(e)
@temporalio.activity.defn
@@ -564,8 +566,7 @@ async def run_local_evaluation_activity(inputs: RunLocalEvaluationInputs) -> Loc
)
except Exception as error:
# Typed so the workflow counts one emit failure per run, after retries end.
- raise ApplicationError(
- "Failed to emit $ai_evaluation", type=EMIT_EVALUATION_EVENT_FAILED_ERROR_TYPE
- ) from error
+ error_class = ApplicationError if is_last_attempt() else NonReportableApplicationError
+ raise error_class("Failed to emit $ai_evaluation", type=EMIT_EVALUATION_EVENT_FAILED_ERROR_TYPE) from error
emitted = True
return LocalEvaluationOutcome(evaluation=evaluation, result=result, emitted=emitted)
diff --git a/posthog/temporal/ai_observability/run_aggregate_evaluation.py b/posthog/temporal/ai_observability/run_aggregate_evaluation.py
index 62d593eb72ad..0a3d1002c7b2 100644
--- a/posthog/temporal/ai_observability/run_aggregate_evaluation.py
+++ b/posthog/temporal/ai_observability/run_aggregate_evaluation.py
@@ -65,6 +65,11 @@ from posthog.temporal.ai_observability.run_trace_evaluation import (
execute_trace_hog_eval_activity,
execute_trace_llm_judge_activity,
)
+from posthog.temporal.ai_observability.team_capture import (
+ EMIT_EVENT_RETRY_POLICY,
+ EMIT_EVENT_SCHEDULE_TO_CLOSE_TIMEOUT,
+ EMIT_EVENT_START_TO_CLOSE_TIMEOUT,
+)
from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.common.utils import close_db_connections
@@ -704,8 +709,9 @@ class RunAggregateEvaluationWorkflow(PostHogWorkflow):
backfill_id=inputs.backfill_id,
event_timestamp=inputs.anchor_timestamp,
),
- schedule_to_close_timeout=timedelta(seconds=30),
- retry_policy=RetryPolicy(maximum_attempts=3),
+ start_to_close_timeout=EMIT_EVENT_START_TO_CLOSE_TIMEOUT,
+ schedule_to_close_timeout=EMIT_EVENT_SCHEDULE_TO_CLOSE_TIMEOUT,
+ retry_policy=EMIT_EVENT_RETRY_POLICY,
)
except Exception:
increment_errors("emit_evaluation_event_failed", provider=result.get("provider"))
diff --git a/posthog/temporal/ai_observability/run_evaluation.py b/posthog/temporal/ai_observability/run_evaluation.py
index bdc9e456c375..57ed366011f7 100644
--- a/posthog/temporal/ai_observability/run_evaluation.py
+++ b/posthog/temporal/ai_observability/run_evaluation.py
@@ -48,6 +48,11 @@ from posthog.temporal.ai_observability.evaluation_workflow_activities import (
update_key_state_activity,
)
from posthog.temporal.ai_observability.metrics import increment_errors
+from posthog.temporal.ai_observability.team_capture import (
+ EMIT_EVENT_RETRY_POLICY,
+ HYDRATING_EMIT_SCHEDULE_TO_CLOSE_TIMEOUT,
+ HYDRATING_EMIT_START_TO_CLOSE_TIMEOUT,
+)
from posthog.temporal.common.base import PostHogWorkflow
from products.ai_observability.backend.models.provider_keys import LLMProviderKey
@@ -292,12 +297,15 @@ class RunEvaluationWorkflow(PostHogWorkflow):
start_time=start_time,
backfill_id=inputs.backfill_id,
),
- start_to_close_timeout=timedelta(seconds=120),
+ start_to_close_timeout=HYDRATING_EMIT_START_TO_CLOSE_TIMEOUT,
# Total deadline including queue wait: without it a task stuck in the queue
# keeps the workflow RUNNING forever, and USE_EXISTING then blocks every later
# trigger for this (evaluation, event) pair.
- schedule_to_close_timeout=backfill_timeout(recover_backfill, timedelta(minutes=8)),
- retry_policy=backfill_retry_policy(recover_backfill, RetryPolicy(maximum_attempts=3)),
+ schedule_to_close_timeout=backfill_timeout(
+ recover_backfill, HYDRATING_EMIT_SCHEDULE_TO_CLOSE_TIMEOUT
+ ),
+ # A hog or sentiment run is local and deterministic, so repeating it on the emit schedule is cheap.
+ retry_policy=backfill_retry_policy(recover_backfill, EMIT_EVENT_RETRY_POLICY),
)
except temporalio.exceptions.ActivityError as e:
if isinstance(e.cause, ApplicationError) and e.cause.type == EMIT_EVALUATION_EVENT_FAILED_ERROR_TYPE:
@@ -385,9 +393,9 @@ class RunEvaluationWorkflow(PostHogWorkflow):
# The activity reads the generation itself when it was handed a reference, and
# that event can run to several MiB, so it gets the same budget as the local
# evaluation activity rather than the 30s a capture call alone would need.
- start_to_close_timeout=timedelta(seconds=120),
- schedule_to_close_timeout=timedelta(minutes=8),
- retry_policy=RetryPolicy(maximum_attempts=3),
+ start_to_close_timeout=HYDRATING_EMIT_START_TO_CLOSE_TIMEOUT,
+ schedule_to_close_timeout=HYDRATING_EMIT_SCHEDULE_TO_CLOSE_TIMEOUT,
+ retry_policy=EMIT_EVENT_RETRY_POLICY,
)
except Exception:
increment_errors("emit_evaluation_event_failed", provider=result.get("provider"))
diff --git a/posthog/temporal/ai_observability/run_tagger.py b/posthog/temporal/ai_observability/run_tagger.py
index ebd7d9fc30b2..3da7901899d5 100644
--- a/posthog/temporal/ai_observability/run_tagger.py
+++ b/posthog/temporal/ai_observability/run_tagger.py
@@ -21,7 +21,13 @@ from posthog.temporal.ai_observability.evaluation_event_io import (
from posthog.temporal.ai_observability.evaluation_workflow_activities import update_key_state_activity
from posthog.temporal.ai_observability.message_utils import extract_text_from_messages
from posthog.temporal.ai_observability.model_resolution import ResolvedModel, model_spec
-from posthog.temporal.ai_observability.team_capture import capture_ai_internal_for_team
+from posthog.temporal.ai_observability.team_capture import (
+ EMIT_EVENT_RETRY_POLICY,
+ HYDRATING_EMIT_SCHEDULE_TO_CLOSE_TIMEOUT,
+ HYDRATING_EMIT_START_TO_CLOSE_TIMEOUT,
+ capture_ai_internal_for_team,
+ raise_emit_failure,
+)
from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.common.utils import close_db_connections
@@ -612,7 +618,9 @@ async def emit_tagger_event_activity(inputs: EmitTaggerEventInputs) -> None:
if e.is_billing_limit_exceeded:
logger.info("Skipping tag event emission; team over billing quota", team_id=event_data["team_id"])
return
- raise
+ raise_emit_failure(e)
+ except Exception as e:
+ raise_emit_failure(e)
@temporalio.activity.defn
@@ -749,8 +757,9 @@ class RunTaggerWorkflow(PostHogWorkflow):
result=result,
start_time=start_time,
),
- schedule_to_close_timeout=timedelta(minutes=2),
- retry_policy=RetryPolicy(maximum_attempts=3),
+ start_to_close_timeout=HYDRATING_EMIT_START_TO_CLOSE_TIMEOUT,
+ schedule_to_close_timeout=HYDRATING_EMIT_SCHEDULE_TO_CLOSE_TIMEOUT,
+ retry_policy=EMIT_EVENT_RETRY_POLICY,
)
return {
diff --git a/posthog/temporal/ai_observability/run_trace_evaluation.py b/posthog/temporal/ai_observability/run_trace_evaluation.py
index 308a019794c7..735b542d5434 100644
--- a/posthog/temporal/ai_observability/run_trace_evaluation.py
+++ b/posthog/temporal/ai_observability/run_trace_evaluation.py
@@ -70,7 +70,13 @@ from posthog.temporal.ai_observability.run_evaluation import (
handle_llm_judge_activity_error,
handle_terminal_user_error_result,
)
-from posthog.temporal.ai_observability.team_capture import capture_ai_internal_for_team
+from posthog.temporal.ai_observability.team_capture import (
+ EMIT_EVENT_RETRY_POLICY,
+ EMIT_EVENT_SCHEDULE_TO_CLOSE_TIMEOUT,
+ EMIT_EVENT_START_TO_CLOSE_TIMEOUT,
+ capture_ai_internal_for_team,
+ raise_emit_failure,
+)
from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.common.utils import close_db_connections
@@ -841,10 +847,10 @@ async def emit_trace_evaluation_event_activity(inputs: EmitTraceEvaluationEventI
logger.info("Skipping eval event emission; team over billing quota", team_id=inputs.team_id)
return
increment_emit_event_outcome("failed")
- raise
- except Exception:
+ raise_emit_failure(e)
+ except Exception as e:
increment_emit_event_outcome("failed")
- raise
+ raise_emit_failure(e)
@temporalio.workflow.defn(name="run-trace-evaluation")
@@ -940,8 +946,9 @@ class RunTraceEvaluationWorkflow(PostHogWorkflow):
result=result,
start_time=eval_start,
),
- schedule_to_close_timeout=timedelta(seconds=30),
- retry_policy=RetryPolicy(maximum_attempts=3),
+ start_to_close_timeout=EMIT_EVENT_START_TO_CLOSE_TIMEOUT,
+ schedule_to_close_timeout=EMIT_EVENT_SCHEDULE_TO_CLOSE_TIMEOUT,
+ retry_policy=EMIT_EVENT_RETRY_POLICY,
)
except Exception:
increment_errors("emit_evaluation_event_failed", provider=result.get("provider"))
diff --git a/posthog/temporal/ai_observability/team_capture.py b/posthog/temporal/ai_observability/team_capture.py
index 7d160dccc6eb..24bc237803f9 100644
--- a/posthog/temporal/ai_observability/team_capture.py
+++ b/posthog/temporal/ai_observability/team_capture.py
@@ -10,12 +10,16 @@ once per team per TTL.
import time
import threading
-from datetime import datetime
+from datetime import datetime, timedelta
from http import HTTPStatus
-from typing import Any
+from typing import Any, NoReturn
+
+from temporalio.common import RetryPolicy
from posthog.api.capture import CaptureInternalError, capture_ai_internal
from posthog.models.team import Team
+from posthog.temporal.ai_observability.retries import is_last_attempt
+from posthog.temporal.common.errors import NonReportableApplicationError
# Also bounds how long emits keep using a rotated token: capture-rs accepts stale tokens at
# the edge, so rotation is only picked up once this TTL lapses.
@@ -25,6 +29,21 @@ _MAX_CACHED_TEAMS = 10_000
_AUTH_REJECTED_STATUSES = (HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN)
+# A capture-ai restart or DNS blip outlasts the urllib3 retries inside capture_ai_internal. The
+# attempts wait 5s, 10s, 20s and 40s between them, so the last one starts over a minute after the first.
+EMIT_EVENT_RETRY_POLICY = RetryPolicy(
+ initial_interval=timedelta(seconds=5),
+ backoff_coefficient=2.0,
+ maximum_attempts=5,
+)
+# Each schedule_to_close budget fits every attempt running to its start_to_close timeout plus the
+# backoff, so the attempt count ends retries and `raise_emit_failure` can tell the final attempt.
+EMIT_EVENT_START_TO_CLOSE_TIMEOUT = timedelta(seconds=30)
+EMIT_EVENT_SCHEDULE_TO_CLOSE_TIMEOUT = timedelta(minutes=4)
+# For emits that first read their generation, which can run to several MiB.
+HYDRATING_EMIT_START_TO_CLOSE_TIMEOUT = timedelta(seconds=120)
+HYDRATING_EMIT_SCHEDULE_TO_CLOSE_TIMEOUT = timedelta(minutes=12)
+
_lock = threading.Lock()
_cache: dict[int, tuple[float, str]] = {}
@@ -33,6 +52,13 @@ class TeamNotFound(Exception):
pass
+def raise_emit_failure(error: Exception) -> NoReturn:
+ """Re-raise an emit failure, kept out of error tracking while Temporal still has attempts left."""
+ if is_last_attempt():
+ raise error
+ raise NonReportableApplicationError(str(error), type=type(error).__name__) from error
+
+
def get_team_api_token(team_id: int) -> str:
"""Return the team's API token, reading Postgres at most once per TTL per worker process."""
now = time.monotonic()
diff --git a/posthog/temporal/ai_observability/test_team_capture.py b/posthog/temporal/ai_observability/test_team_capture.py
index e4b68ee62003..56bedaa4aa83 100644
--- a/posthog/temporal/ai_observability/test_team_capture.py
+++ b/posthog/temporal/ai_observability/test_team_capture.py
@@ -1,19 +1,30 @@
import time
-from datetime import UTC, datetime
+import dataclasses
+from datetime import UTC, datetime, timedelta
+import time_machine
from posthog.test.base import BaseTest
from unittest.mock import MagicMock, patch
+from django.test import SimpleTestCase
+
from parameterized import parameterized
+from temporalio.testing import ActivityEnvironment
from posthog.api.capture import CaptureInternalError
from posthog.models.team import Team
from posthog.temporal.ai_observability.team_capture import (
+ EMIT_EVENT_RETRY_POLICY,
+ EMIT_EVENT_SCHEDULE_TO_CLOSE_TIMEOUT,
TOKEN_CACHE_TTL_SECONDS,
TeamNotFound,
capture_ai_internal_for_team,
get_team_api_token,
+ raise_emit_failure,
)
+from posthog.temporal.common.errors import NonReportableApplicationError
+
+NOW = datetime(2026, 1, 1, tzinfo=UTC)
def _emit(team_id: int) -> None:
@@ -87,3 +98,28 @@ class TestTeamCapture(BaseTest):
):
with self.assertNumQueries(0):
_emit(self.team.id)
+
+
+class TestRaiseEmitFailure(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("retry_left", 1, 3600, NonReportableApplicationError),
+ ("final_attempt", EMIT_EVENT_RETRY_POLICY.maximum_attempts, 3600, CaptureInternalError),
+ ("backoff_lands_past_deadline", 1, 2, CaptureInternalError),
+ ]
+ )
+ def test_only_the_final_attempt_raises_a_reportable_error(
+ self, _name, attempt, seconds_to_deadline, expected_error
+ ):
+ schedule_to_close = EMIT_EVENT_SCHEDULE_TO_CLOSE_TIMEOUT
+ env = ActivityEnvironment()
+ env.info = dataclasses.replace(
+ env.info,
+ attempt=attempt,
+ retry_policy=EMIT_EVENT_RETRY_POLICY,
+ schedule_to_close_timeout=schedule_to_close,
+ scheduled_time=NOW - schedule_to_close + timedelta(seconds=seconds_to_deadline),
+ )
+ with time_machine.travel(NOW, tick=False), self.assertRaises(Exception) as ctx:
+ env.run(raise_emit_failure, CaptureInternalError("blip"))
+ assert type(ctx.exception) is expected_error