:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(aio): suppress expected backfill generation misses (#122568)
Bernat Torres5fe43b3898fc
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.mdindex eedc7c90fc2a..52e6e3efb816 100644--- a/docs/internal/ai-observability-judge-inputs.md+++ b/docs/internal/ai-observability-judge-inputs.md@@ -195,6 +195,10 @@ This excludes the calendar day whose AI payloads have expired under the 30-day rRanges ending at or before that boundary are rejected.Existing backfills keep their stored windows; **Retry remaining** applies the boundary when it creates a new backfill.+Generation references get up to three lookup attempts.+A final miss for a backfill reference fails the run without reporting an exception to error tracking.+Live references marked `awaiting_ingestion` wait 15 seconds between misses and still report the final miss.+A backfill that stops on its own shows as **Interrupted**, with a stop reason.A backfill a user stops shows as **Cancelled**.Backfills read a page of units (500 by default, bounded between 1 and 1,000) and wait for every outcome before advancing progress.diff --git a/posthog/temporal/ai_observability/evaluation_event_io.py b/posthog/temporal/ai_observability/evaluation_event_io.pyindex f4022005251e..49eb498dd181 100644--- a/posthog/temporal/ai_observability/evaluation_event_io.py+++ b/posthog/temporal/ai_observability/evaluation_event_io.py@@ -91,11 +91,11 @@ def hydrate_event_reference(event_data: dict[str, Any]) -> dict[str, Any]:)if event is None:if temporalio.activity.in_activity() and temporalio.activity.info().attempt < GENERATION_NOT_FOUND_MAX_ATTEMPTS:- # Only the last miss reaches error tracking: an earlier one is usually the lag above.raise NonReportableApplicationError("Generation not found",type="generation_not_found",next_retry_delay=GENERATION_NOT_FOUND_RETRY_DELAY if event_data.get("awaiting_ingestion") else None,)- raise ApplicationError("Generation not found", type="generation_not_found", non_retryable=True)+ error_class = ApplicationError if event_data.get("awaiting_ingestion") else NonReportableApplicationError+ raise error_class("Generation not found", type="generation_not_found", non_retryable=True)return eventdiff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.pyindex bf2b79264a54..4fb7c7ec3d8e 100644--- a/posthog/temporal/ai_observability/test_run_evaluation.py+++ b/posthog/temporal/ai_observability/test_run_evaluation.py@@ -31,6 +31,7 @@ from posthog.models import Organization, OrganizationMembership, Team, Userfrom posthog.temporal.ai_observability.sentiment.extraction import truncate_to_head_tailfrom posthog.temporal.ai_observability.sentiment.schema import SentimentResultfrom posthog.temporal.common.errors import NonReportableError+from posthog.temporal.common.posthog_client import is_expected_activity_failurefrom products.access_control.backend.models.access_control import AccessControlfrom products.ai_observability.backend.llm.errors import (@@ -2306,7 +2307,7 @@ class TestRunEvaluationWorkflow:assert raised.value.type == "generation_not_found"assert raised.value.non_retryable is not retryable- assert isinstance(raised.value, NonReportableError) is retryable+ assert is_expected_activity_failure(raised.value) is (retryable or not live)expected_delay = GENERATION_NOT_FOUND_RETRY_DELAY if retryable and live else Noneassert raised.value.next_retry_delay == expected_delay