Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(aio): align backfill windows with ai event retention (#121990)

Bernat Torres92d3499ba42b
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index 890d5644fb73..80eee6a04d6d 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -183,6 +183,11 @@ The playground keeps the provider's explanation so users can correct the setting
## Backfill recovery
+Estimates and new backfills clamp their start to UTC midnight 29 days before the current UTC date.
+This excludes the calendar day whose AI payloads have expired under the 30-day retention policy.
+Ranges ending at or before that boundary are rejected.
+Existing backfills keep their stored windows; **Retry remaining** applies the boundary when it creates a new backfill.
+
A backfill that stops on its own shows as **Interrupted**, with a stop reason.
A backfill a user stops shows as **Cancelled**.
Backfills read a page of units (500 by default, bounded between 1 and 1,000) and wait for every outcome before advancing progress.
diff --git a/products/ai_observability/backend/api/evaluation_backfills.py b/products/ai_observability/backend/api/evaluation_backfills.py
index 92d90aec340e..318e2454eb19 100644
--- a/products/ai_observability/backend/api/evaluation_backfills.py
+++ b/products/ai_observability/backend/api/evaluation_backfills.py
@@ -348,7 +348,11 @@ class EvaluationBackfillViewSet(
f"{_duration_label(settle_hold - timedelta(seconds=INGESTION_LAG_MARGIN_SECONDS))}, so a "
"backfill has to end before that. Try a range that ends earlier."
)
- window_start: datetime = max(data["window_start"], now - timedelta(days=AI_EVENTS_RETENTION_DAYS))
+ # ai_events expires at toDate(timestamp) + retention_days, so exclude the whole expired day.
+ retention_start = now.replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(
+ days=AI_EVENTS_RETENTION_DAYS - 1
+ )
+ window_start: datetime = max(data["window_start"], retention_start)
if window_start >= window_end:
raise ValidationError(
f"The range must end in the past and start within the last {AI_EVENTS_RETENTION_DAYS} days."
diff --git a/products/ai_observability/backend/api/test/test_evaluation_backfills.py b/products/ai_observability/backend/api/test/test_evaluation_backfills.py
index 886a7c56c688..1f77ebd14640 100644
--- a/products/ai_observability/backend/api/test/test_evaluation_backfills.py
+++ b/products/ai_observability/backend/api/test/test_evaluation_backfills.py
@@ -2,6 +2,7 @@ from datetime import datetime, timedelta
from types import SimpleNamespace
from typing import cast
+import time_machine
from posthog.test.base import APIBaseTest
from unittest.mock import AsyncMock, MagicMock, patch
@@ -26,7 +27,6 @@ from posthog.rate_limit import (
AIObservabilityBackfillEstimateSustainedThrottle,
AIObservabilityBackfillEstimateThrottle,
)
-from posthog.temporal.ai_observability.run_session_evaluation import AI_EVENTS_RETENTION_DAYS
from products.access_control.backend.models.access_control import AccessControl
from products.ai_observability.backend.api.evaluation_backfills import (
@@ -376,22 +376,31 @@ class TestEvaluationBackfillsApi(APIBaseTest):
stale.refresh_from_db()
assert stale.status == EvaluationBackfillStatus.RUNNING
- @parameterized.expand(["start_after_end", "entirely_in_future"])
- def test_create_rejects_bad_windows(self, case):
+ @parameterized.expand(["start_after_end", "entirely_in_future", "expired_retention_day"])
+ @time_machine.travel("2024-03-01T12:00:00Z", tick=False)
+ @patch(f"{API_MODULE}.count_backfill_candidates", return_value=_scope(3))
+ @patch(f"{API_MODULE}.sync_connect")
+ def test_create_rejects_bad_windows(self, case: str, connect: MagicMock, _count: MagicMock) -> None:
+ connect.return_value = _temporal_client()
now = timezone.now()
windows = {
"start_after_end": (now, now - timedelta(days=1)),
"entirely_in_future": (now + timedelta(days=1), now + timedelta(days=2)),
+ "expired_retention_day": (
+ datetime.fromisoformat("2024-01-31T18:00:00+00:00"),
+ datetime.fromisoformat("2024-02-01T00:00:00+00:00"),
+ ),
}
window_start, window_end = windows[case]
- response = self.client.post(
- f"{self.url}/",
- {"window_start": window_start.isoformat(), "window_end": window_end.isoformat()},
- format="json",
- )
+ for path in (f"{self.url}/estimate/", f"{self.url}/"):
+ response = self.client.post(
+ path,
+ {"window_start": window_start.isoformat(), "window_end": window_end.isoformat()},
+ format="json",
+ )
- assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
+ assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json()
assert EvaluationBackfill.objects.unscoped().count() == 0
@parameterized.expand(["evaluation_has_no_conditions", "explicit_empty_list"])
@@ -409,28 +418,36 @@ class TestEvaluationBackfillsApi(APIBaseTest):
assert "at least one condition set" in response.json()["detail"]
assert EvaluationBackfill.objects.unscoped().count() == 0
+ @parameterized.expand(
+ [
+ ("2024-02-29T23:59:59Z", "2023-12-01T00:00:00+00:00", "2024-01-31T00:00:00+00:00"),
+ ("2024-03-01T00:00:00Z", "2023-12-01T00:00:00+00:00", "2024-02-01T00:00:00+00:00"),
+ ("2024-03-01T12:00:00Z", "2023-12-01T00:00:00+00:00", "2024-02-01T00:00:00+00:00"),
+ ("2024-03-01T12:00:00Z", "2024-02-15T07:30:00+05:30", "2024-02-15T02:00:00+00:00"),
+ ]
+ )
@patch(f"{API_MODULE}.count_backfill_candidates", return_value=_scope(3))
@patch(f"{API_MODULE}.sync_connect")
- def test_clamps_a_window_wider_than_retention(self, connect, _count):
+ def test_clamps_a_window_to_ai_events_retention(
+ self, now_iso: str, requested_start: str, retained_start: str, connect: MagicMock, _count: MagicMock
+ ) -> None:
connect.return_value = _temporal_client()
- now = timezone.now()
- body = _body(
- window_start=(now - timedelta(days=90)).isoformat(),
- window_end=(now + timedelta(days=1)).isoformat(),
- )
-
- estimate = self.client.post(f"{self.url}/estimate/", body, format="json")
- assert estimate.status_code == status.HTTP_200_OK, estimate.json()
- expected_end = now - BACKFILL_AI_EVENTS_LAG
- assert abs(datetime.fromisoformat(estimate.json()["window_end"]) - expected_end) < timedelta(seconds=5)
- expected_start = now - timedelta(days=AI_EVENTS_RETENTION_DAYS)
- assert abs(datetime.fromisoformat(estimate.json()["window_start"]) - expected_start) < timedelta(seconds=5)
-
- created = self.client.post(f"{self.url}/", body, format="json")
- assert created.status_code == status.HTTP_201_CREATED, created.json()
- row = EvaluationBackfill.objects.unscoped().get(pk=created.json()["id"])
- assert abs(row.window_end - expected_end) < timedelta(seconds=5)
- assert abs(row.window_start - expected_start) < timedelta(seconds=5)
+ with time_machine.travel(now_iso, tick=False):
+ now = timezone.now()
+ body = _body(window_start=requested_start, window_end=(now + timedelta(days=1)).isoformat())
+
+ estimate = self.client.post(f"{self.url}/estimate/", body, format="json")
+ assert estimate.status_code == status.HTTP_200_OK, estimate.json()
+ expected_end = now - BACKFILL_AI_EVENTS_LAG
+ expected_start = datetime.fromisoformat(retained_start)
+ assert datetime.fromisoformat(estimate.json()["window_end"]) == expected_end
+ assert datetime.fromisoformat(estimate.json()["window_start"]) == expected_start
+
+ created = self.client.post(f"{self.url}/", body, format="json")
+ assert created.status_code == status.HTTP_201_CREATED, created.json()
+ row = EvaluationBackfill.objects.unscoped().get(pk=created.json()["id"])
+ assert row.window_end == expected_end
+ assert row.window_start == expected_start
@parameterized.expand(
[