Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(aio): suppress expected backfill generation misses (#122568)

Bernat Torres5fe43b3898fc
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index eedc7c90fc2a..52e6e3efb816 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -195,6 +195,10 @@ This excludes the calendar day whose AI payloads have expired under the 30-day r
Ranges ending at or before that boundary are rejected.
Existing backfills keep their stored windows; **Retry remaining** applies the boundary when it creates a new backfill.
+Generation references get up to three lookup attempts.
+A final miss for a backfill reference fails the run without reporting an exception to error tracking.
+Live references marked `awaiting_ingestion` wait 15 seconds between misses and still report the final miss.
+
A backfill that stops on its own shows as **Interrupted**, with a stop reason.
A backfill a user stops shows as **Cancelled**.
Backfills read a page of units (500 by default, bounded between 1 and 1,000) and wait for every outcome before advancing progress.
diff --git a/posthog/temporal/ai_observability/evaluation_event_io.py b/posthog/temporal/ai_observability/evaluation_event_io.py
index f4022005251e..49eb498dd181 100644
--- a/posthog/temporal/ai_observability/evaluation_event_io.py
+++ b/posthog/temporal/ai_observability/evaluation_event_io.py
@@ -91,11 +91,11 @@ def hydrate_event_reference(event_data: dict[str, Any]) -> dict[str, Any]:
)
if event is None:
if temporalio.activity.in_activity() and temporalio.activity.info().attempt < GENERATION_NOT_FOUND_MAX_ATTEMPTS:
- # Only the last miss reaches error tracking: an earlier one is usually the lag above.
raise NonReportableApplicationError(
"Generation not found",
type="generation_not_found",
next_retry_delay=GENERATION_NOT_FOUND_RETRY_DELAY if event_data.get("awaiting_ingestion") else None,
)
- raise ApplicationError("Generation not found", type="generation_not_found", non_retryable=True)
+ error_class = ApplicationError if event_data.get("awaiting_ingestion") else NonReportableApplicationError
+ raise error_class("Generation not found", type="generation_not_found", non_retryable=True)
return event
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index bf2b79264a54..4fb7c7ec3d8e 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -31,6 +31,7 @@ from posthog.models import Organization, OrganizationMembership, Team, User
from posthog.temporal.ai_observability.sentiment.extraction import truncate_to_head_tail
from posthog.temporal.ai_observability.sentiment.schema import SentimentResult
from posthog.temporal.common.errors import NonReportableError
+from posthog.temporal.common.posthog_client import is_expected_activity_failure
from products.access_control.backend.models.access_control import AccessControl
from products.ai_observability.backend.llm.errors import (
@@ -2306,7 +2307,7 @@ class TestRunEvaluationWorkflow:
assert raised.value.type == "generation_not_found"
assert raised.value.non_retryable is not retryable
- assert isinstance(raised.value, NonReportableError) is retryable
+ assert is_expected_activity_failure(raised.value) is (retryable or not live)
expected_delay = GENERATION_NOT_FOUND_RETRY_DELAY if retryable and live else None
assert raised.value.next_retry_delay == expected_delay