:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(aio): align backfill windows with ai event retention (#121990)
Bernat Torres92d3499ba42b
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.mdindex 890d5644fb73..80eee6a04d6d 100644--- a/docs/internal/ai-observability-judge-inputs.md+++ b/docs/internal/ai-observability-judge-inputs.md@@ -183,6 +183,11 @@ The playground keeps the provider's explanation so users can correct the setting## Backfill recovery+Estimates and new backfills clamp their start to UTC midnight 29 days before the current UTC date.+This excludes the calendar day whose AI payloads have expired under the 30-day retention policy.+Ranges ending at or before that boundary are rejected.+Existing backfills keep their stored windows; **Retry remaining** applies the boundary when it creates a new backfill.+A backfill that stops on its own shows as **Interrupted**, with a stop reason.A backfill a user stops shows as **Cancelled**.Backfills read a page of units (500 by default, bounded between 1 and 1,000) and wait for every outcome before advancing progress.diff --git a/products/ai_observability/backend/api/evaluation_backfills.py b/products/ai_observability/backend/api/evaluation_backfills.pyindex 92d90aec340e..318e2454eb19 100644--- a/products/ai_observability/backend/api/evaluation_backfills.py+++ b/products/ai_observability/backend/api/evaluation_backfills.py@@ -348,7 +348,11 @@ class EvaluationBackfillViewSet(f"{_duration_label(settle_hold - timedelta(seconds=INGESTION_LAG_MARGIN_SECONDS))}, so a ""backfill has to end before that. Try a range that ends earlier.")- window_start: datetime = max(data["window_start"], now - timedelta(days=AI_EVENTS_RETENTION_DAYS))+ # ai_events expires at toDate(timestamp) + retention_days, so exclude the whole expired day.+ retention_start = now.replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(+ days=AI_EVENTS_RETENTION_DAYS - 1+ )+ window_start: datetime = max(data["window_start"], retention_start)if window_start >= window_end:raise ValidationError(f"The range must end in the past and start within the last {AI_EVENTS_RETENTION_DAYS} days."diff --git a/products/ai_observability/backend/api/test/test_evaluation_backfills.py b/products/ai_observability/backend/api/test/test_evaluation_backfills.pyindex 886a7c56c688..1f77ebd14640 100644--- a/products/ai_observability/backend/api/test/test_evaluation_backfills.py+++ b/products/ai_observability/backend/api/test/test_evaluation_backfills.py@@ -2,6 +2,7 @@ from datetime import datetime, timedeltafrom types import SimpleNamespacefrom typing import cast+import time_machinefrom posthog.test.base import APIBaseTestfrom unittest.mock import AsyncMock, MagicMock, patch@@ -26,7 +27,6 @@ from posthog.rate_limit import (AIObservabilityBackfillEstimateSustainedThrottle,AIObservabilityBackfillEstimateThrottle,)-from posthog.temporal.ai_observability.run_session_evaluation import AI_EVENTS_RETENTION_DAYSfrom products.access_control.backend.models.access_control import AccessControlfrom products.ai_observability.backend.api.evaluation_backfills import (@@ -376,22 +376,31 @@ class TestEvaluationBackfillsApi(APIBaseTest):stale.refresh_from_db()assert stale.status == EvaluationBackfillStatus.RUNNING- @parameterized.expand(["start_after_end", "entirely_in_future"])- def test_create_rejects_bad_windows(self, case):+ @parameterized.expand(["start_after_end", "entirely_in_future", "expired_retention_day"])+ @time_machine.travel("2024-03-01T12:00:00Z", tick=False)+ @patch(f"{API_MODULE}.count_backfill_candidates", return_value=_scope(3))+ @patch(f"{API_MODULE}.sync_connect")+ def test_create_rejects_bad_windows(self, case: str, connect: MagicMock, _count: MagicMock) -> None:+ connect.return_value = _temporal_client()now = timezone.now()windows = {"start_after_end": (now, now - timedelta(days=1)),"entirely_in_future": (now + timedelta(days=1), now + timedelta(days=2)),+ "expired_retention_day": (+ datetime.fromisoformat("2024-01-31T18:00:00+00:00"),+ datetime.fromisoformat("2024-02-01T00:00:00+00:00"),+ ),}window_start, window_end = windows[case]- response = self.client.post(- f"{self.url}/",- {"window_start": window_start.isoformat(), "window_end": window_end.isoformat()},- format="json",- )+ for path in (f"{self.url}/estimate/", f"{self.url}/"):+ response = self.client.post(+ path,+ {"window_start": window_start.isoformat(), "window_end": window_end.isoformat()},+ format="json",+ )- assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()+ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()assert EvaluationBackfill.objects.unscoped().count() == 0@parameterized.expand(["evaluation_has_no_conditions", "explicit_empty_list"])@@ -409,28 +418,36 @@ class TestEvaluationBackfillsApi(APIBaseTest):assert "at least one condition set" in response.json()["detail"]assert EvaluationBackfill.objects.unscoped().count() == 0+ @parameterized.expand(+ [+ ("2024-02-29T23:59:59Z", "2023-12-01T00:00:00+00:00", "2024-01-31T00:00:00+00:00"),+ ("2024-03-01T00:00:00Z", "2023-12-01T00:00:00+00:00", "2024-02-01T00:00:00+00:00"),+ ("2024-03-01T12:00:00Z", "2023-12-01T00:00:00+00:00", "2024-02-01T00:00:00+00:00"),+ ("2024-03-01T12:00:00Z", "2024-02-15T07:30:00+05:30", "2024-02-15T02:00:00+00:00"),+ ]+ )@patch(f"{API_MODULE}.count_backfill_candidates", return_value=_scope(3))@patch(f"{API_MODULE}.sync_connect")- def test_clamps_a_window_wider_than_retention(self, connect, _count):+ def test_clamps_a_window_to_ai_events_retention(+ self, now_iso: str, requested_start: str, retained_start: str, connect: MagicMock, _count: MagicMock+ ) -> None:connect.return_value = _temporal_client()- now = timezone.now()- body = _body(- window_start=(now - timedelta(days=90)).isoformat(),- window_end=(now + timedelta(days=1)).isoformat(),- )-- estimate = self.client.post(f"{self.url}/estimate/", body, format="json")- assert estimate.status_code == status.HTTP_200_OK, estimate.json()- expected_end = now - BACKFILL_AI_EVENTS_LAG- assert abs(datetime.fromisoformat(estimate.json()["window_end"]) - expected_end) < timedelta(seconds=5)- expected_start = now - timedelta(days=AI_EVENTS_RETENTION_DAYS)- assert abs(datetime.fromisoformat(estimate.json()["window_start"]) - expected_start) < timedelta(seconds=5)-- created = self.client.post(f"{self.url}/", body, format="json")- assert created.status_code == status.HTTP_201_CREATED, created.json()- row = EvaluationBackfill.objects.unscoped().get(pk=created.json()["id"])- assert abs(row.window_end - expected_end) < timedelta(seconds=5)- assert abs(row.window_start - expected_start) < timedelta(seconds=5)+ with time_machine.travel(now_iso, tick=False):+ now = timezone.now()+ body = _body(window_start=requested_start, window_end=(now + timedelta(days=1)).isoformat())++ estimate = self.client.post(f"{self.url}/estimate/", body, format="json")+ assert estimate.status_code == status.HTTP_200_OK, estimate.json()+ expected_end = now - BACKFILL_AI_EVENTS_LAG+ expected_start = datetime.fromisoformat(retained_start)+ assert datetime.fromisoformat(estimate.json()["window_end"]) == expected_end+ assert datetime.fromisoformat(estimate.json()["window_start"]) == expected_start++ created = self.client.post(f"{self.url}/", body, format="json")+ assert created.status_code == status.HTTP_201_CREATED, created.json()+ row = EvaluationBackfill.objects.unscoped().get(pk=created.json()["id"])+ assert row.window_end == expected_end+ assert row.window_start == expected_start@parameterized.expand([