:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(aio): expire offline evaluation payloads after 30 days (#126467)
Radu Raiceae50633dd0a7a
diff --git a/docs/internal/ai-offline-evaluation-reporting.md b/docs/internal/ai-offline-evaluation-reporting.mdindex c74f74c302a3..919779c2216d 100644--- a/docs/internal/ai-offline-evaluation-reporting.md+++ b/docs/internal/ai-offline-evaluation-reporting.md@@ -401,7 +401,7 @@ Local and external datasets do not require hosted links; they can use the `*_ideLarge payloads have separate storage and 30-day deadlines anchored to first acceptance.Retries neither extend deadlines nor restore deleted payloads.-Automatic payload deletion is not enabled by these endpoints.+An hourly Celery sweep deletes payload rows past their deadline and marks their owners expired in the same transaction, preserving identities, results, summaries, and usage. Payloads remain readable until the sweep removes them.### Product usage events@@ -652,8 +652,10 @@ Payload detail responses include `available` and `data`, preserving omitted prop`not_provided` means the caller omitted the payload; `expired` means it was removed while its owner was retained.An unavailable payload has null `data`, while durable identities, results, and summaries remain readable.-A deadline alone does not mean a cleanup worker has removed the payload.-Automatic deletion remains separate work.+Passing the deadline makes a payload eligible for cleanup. It remains available until an hourly sweep removes it; delayed runs or a backlog can extend availability.+The sweep processes at most 20 batches of 500 candidates per payload type, alternating item and result payloads, with a three-minute work budget checked before each environment transaction. Each environment's batch commits independently; retries and overlapping runs are safe. Existing overdue payloads are included, and a backlog drains over subsequent runs.++Each run emits `offline evaluation payload retention run` using scoped capture, including empty runs and handled failures. Properties include `run_id`, `status`, `batches` (committed environment batches), `item_payloads_expired`, `result_payloads_expired`, `payloads_expired`, `budget_exhausted`, and `duration_seconds`. Counts reflect committed work, and the same properties are logged. Event delivery is best-effort; a hard-killed worker cannot emit its completion event.Expired input/output is not reconstructed from linked datasets or traces, and missing links do not prevent experiment reads.Opening those resources requires their own permissions.diff --git a/posthog/tasks/scheduled.py b/posthog/tasks/scheduled.pyindex 733f7f173cf7..1af0bbea153f 100644--- a/posthog/tasks/scheduled.py+++ b/posthog/tasks/scheduled.py@@ -78,6 +78,7 @@ from posthog.utils import get_crontab, get_instance_regionfrom products.account_audits.backend.facade.tasks import reconcile_account_auditsfrom products.aeo.backend.facade.tasks import run_aeo_citation_checks_task+from products.ai_observability.backend.facade.tasks import expire_offline_evaluation_payloadsfrom products.ai_training.backend.facade.api import privacy_enabledfrom products.ai_training.backend.facade.tasks import process_ai_training_privacy_requestsfrom products.approvals.backend.tasks import (@@ -273,6 +274,12 @@ def add_periodic_task_with_expiry(def setup_periodic_tasks(sender: Celery, **kwargs: Any) -> None:+ add_periodic_task_with_expiry(+ sender,+ crontab(minute="23"),+ expire_offline_evaluation_payloads.s(),+ name="expire offline evaluation payloads",+ )add_periodic_task_with_expiry(sender, crontab(minute="*"), reconcile_account_audits.s(), name="reconcile-account-audits")diff --git a/products/ai_observability/backend/api/offline_experiment_read_serializers.py b/products/ai_observability/backend/api/offline_experiment_read_serializers.pyindex 8316e4b74af8..a072ef75becc 100644--- a/products/ai_observability/backend/api/offline_experiment_read_serializers.py+++ b/products/ai_observability/backend/api/offline_experiment_read_serializers.py@@ -299,7 +299,7 @@ class OfflineResultFieldsSerializer(serializers.Serializer):accepted_at = serializers.DateTimeField(help_text="Original server acceptance time.")payload_state = serializers.ChoiceField(choices=PayloadState.choices, help_text="Result payload storage state.")payload_expires_at = serializers.DateTimeField(- allow_null=True, help_text="Payload retention deadline; cleanup is not yet enabled."+ allow_null=True, help_text="Payload expires 30 days after first acceptance; deletion runs hourly.")@@ -344,7 +344,7 @@ class OfflineItemReadSerializer(serializers.Serializer):accepted_at = serializers.DateTimeField(help_text="Original server acceptance time.")payload_state = serializers.ChoiceField(choices=PayloadState.choices, help_text="Item payload storage state.")payload_expires_at = serializers.DateTimeField(- allow_null=True, help_text="Payload retention deadline; cleanup is not yet enabled."+ allow_null=True, help_text="Payload expires 30 days after first acceptance; deletion runs hourly.")results = OfflineResultCellSerializer(many=True, help_text="Cells for explicitly selected scorer versions; empty when none selected."diff --git a/products/ai_observability/backend/facade/tasks.py b/products/ai_observability/backend/facade/tasks.pynew file mode 100644index 000000000000..109cd1364bcb--- /dev/null+++ b/products/ai_observability/backend/facade/tasks.py@@ -0,0 +1,3 @@+from products.ai_observability.backend.tasks.offline_payload_retention import expire_offline_evaluation_payloads++__all__ = ["expire_offline_evaluation_payloads"]diff --git a/products/ai_observability/backend/migrations/0070_offline_payload_expiry_indexes.py b/products/ai_observability/backend/migrations/0070_offline_payload_expiry_indexes.pynew file mode 100644index 000000000000..f334cf9dd991--- /dev/null+++ b/products/ai_observability/backend/migrations/0070_offline_payload_expiry_indexes.py@@ -0,0 +1,28 @@+from django.db import migrations, models++from posthog.migration_helpers import SafeAddIndexConcurrently+++class Migration(migrations.Migration):+ atomic = False++ dependencies = [("ai_observability", "0069_offline_evaluation_usage_ledger")]++ operations = [+ SafeAddIndexConcurrently(+ model_name="offlineexperimentitem",+ index=models.Index(+ fields=["payload_expires_at", "id"],+ condition=models.Q(payload_state="available"),+ name="aio_item_payload_expiry_idx",+ ),+ ),+ SafeAddIndexConcurrently(+ model_name="offlineevaluationresult",+ index=models.Index(+ fields=["payload_expires_at", "id"],+ condition=models.Q(payload_state="available"),+ name="aio_result_payload_expiry_idx",+ ),+ ),+ ]diff --git a/products/ai_observability/backend/migrations/max_migration.txt b/products/ai_observability/backend/migrations/max_migration.txtindex 5ec97173b6aa..0317898d12ac 100644--- a/products/ai_observability/backend/migrations/max_migration.txt+++ b/products/ai_observability/backend/migrations/max_migration.txt@@ -1 +1 @@-0069_offline_evaluation_usage_ledger+0070_offline_payload_expiry_indexesdiff --git a/products/ai_observability/backend/models/offline_evaluations.py b/products/ai_observability/backend/models/offline_evaluations.pyindex b7cc2d4c1425..d9dd31c0ed85 100644--- a/products/ai_observability/backend/models/offline_evaluations.py+++ b/products/ai_observability/backend/models/offline_evaluations.py@@ -144,6 +144,11 @@ class OfflineExperimentItem(OfflinePayloadOwner):]indexes = [models.Index(fields=["experiment", "id"], name="aio_offline_item_exp_idx"),+ models.Index(+ fields=["payload_expires_at", "id"],+ condition=Q(payload_state="available"),+ name="aio_item_payload_expiry_idx",+ ),]@@ -216,6 +221,11 @@ class OfflineEvaluationResult(OfflinePayloadOwner, UUIDModel):models.Index(fields=["team", "scorer_version", "-accepted_at", "id"], name="aio_offline_result_history_idx"),+ models.Index(+ fields=["payload_expires_at", "id"],+ condition=Q(payload_state="available"),+ name="aio_result_payload_expiry_idx",+ ),]diff --git a/products/ai_observability/backend/offline_evaluation_read_service.py b/products/ai_observability/backend/offline_evaluation_read_service.pyindex 1e74790f33ef..44f2c760d314 100644--- a/products/ai_observability/backend/offline_evaluation_read_service.py+++ b/products/ai_observability/backend/offline_evaluation_read_service.py@@ -539,14 +539,15 @@ class OfflineEvaluationReadService:def get_item_payload(self, experiment_id: UUID, item_id: UUID) -> OfflinePayloadRead:item = self._require_item(experiment_id, item_id)+ payload_state = item.payload_statepayload = (OfflineExperimentItemPayload.objects.for_team(self.team_id).filter(item_id=item.id).first()- if item.payload_state == PayloadState.AVAILABLE+ if payload_state == PayloadState.AVAILABLEelse None)return OfflinePayloadRead(id=item.id,- payload_state=item.payload_state,+ payload_state=payload_state,payload_expires_at=item.payload_expires_at,available=payload is not None,data=payload.data if payload is not None else None,@@ -556,14 +557,15 @@ class OfflineEvaluationReadService:result = self._results().filter(id=result_id, item__experiment_id=experiment_id).first()if result is None:raise OfflineEvaluationNotFound+ payload_state = result.payload_statepayload = (OfflineEvaluationResultPayload.objects.for_team(self.team_id).filter(result_id=result.id).first()- if result.payload_state == PayloadState.AVAILABLE+ if payload_state == PayloadState.AVAILABLEelse None)return OfflinePayloadRead(id=result.id,- payload_state=result.payload_state,+ payload_state=payload_state,payload_expires_at=result.payload_expires_at,available=payload is not None,data=payload.data if payload is not None else None,diff --git a/products/ai_observability/backend/offline_payload_retention.py b/products/ai_observability/backend/offline_payload_retention.pynew file mode 100644index 000000000000..f0ed2e1e71b4--- /dev/null+++ b/products/ai_observability/backend/offline_payload_retention.py@@ -0,0 +1,86 @@+import time+from collections import defaultdict+from uuid import UUID++from django.db import transaction+from django.utils import timezone++from products.ai_observability.backend.models.offline_evaluations import (+ OfflineEvaluationResult,+ OfflineEvaluationResultPayload,+ OfflineExperimentItem,+ OfflineExperimentItemPayload,+ PayloadState,+)++BATCH_SIZE = 500+MAX_BATCHES_PER_TYPE = 20+TIME_BUDGET_SECONDS = 180+++class OfflinePayloadRetention:+ def __init__(self) -> None:+ self.batches = 0+ self.item_payloads_expired = 0+ self.result_payloads_expired = 0+ self.budget_exhausted = False++ def _expire_batch(+ self,+ owner_model: type[OfflineExperimentItem] | type[OfflineEvaluationResult],+ payload_model: type[OfflineExperimentItemPayload] | type[OfflineEvaluationResultPayload],+ deadline: float,+ ) -> bool:+ cutoff = timezone.now()+ by_team: dict[int, list[UUID]] = defaultdict(list)+ candidates = (+ owner_model.objects.unscoped()+ .filter(payload_state=PayloadState.AVAILABLE, payload_expires_at__lte=cutoff)+ .order_by("payload_expires_at", "id")+ .values_list("id", "team_id")[:BATCH_SIZE]+ )+ for owner_id, team_id in candidates:+ by_team[team_id].append(owner_id)++ progressed = False+ for team_id, ids in by_team.items():+ if time.monotonic() >= deadline:+ self.budget_exhausted = True+ return progressed+ with transaction.atomic():+ owners = owner_model.objects.for_team(team_id).filter(+ id__in=ids, payload_state=PayloadState.AVAILABLE, payload_expires_at__lte=cutoff+ )+ locked_ids = list(owners.select_for_update(skip_locked=True).values_list("id", flat=True))+ if not locked_ids:+ continue+ deleted, _ = payload_model.objects.for_team(team_id).filter(pk__in=locked_ids).delete()+ owners.filter(id__in=locked_ids).update(payload_state=PayloadState.EXPIRED)+ self.batches += 1+ if payload_model is OfflineExperimentItemPayload:+ self.item_payloads_expired += deleted+ else:+ self.result_payloads_expired += deleted+ progressed = True+ return progressed++ def run(self) -> None:+ deadline = time.monotonic() + TIME_BUDGET_SECONDS+ items_remaining = results_remaining = True+ for _ in range(MAX_BATCHES_PER_TYPE):+ for owner_model, payload_model, remaining in (+ (OfflineExperimentItem, OfflineExperimentItemPayload, items_remaining),+ (OfflineEvaluationResult, OfflineEvaluationResultPayload, results_remaining),+ ):+ if time.monotonic() >= deadline:+ self.budget_exhausted = True+ return+ if remaining:+ progressed = self._expire_batch(owner_model, payload_model, deadline)+ if owner_model is OfflineExperimentItem:+ items_remaining = progressed+ else:+ results_remaining = progressed+ if not items_remaining and not results_remaining:+ return+ self.budget_exhausted = Truediff --git a/products/ai_observability/backend/tasks/__init__.py b/products/ai_observability/backend/tasks/__init__.pyindex e69de29bb2d1..b69cd3fdf549 100644--- a/products/ai_observability/backend/tasks/__init__.py+++ b/products/ai_observability/backend/tasks/__init__.py@@ -0,0 +1,3 @@+from .offline_payload_retention import expire_offline_evaluation_payloads++__all__ = ["expire_offline_evaluation_payloads"]diff --git a/products/ai_observability/backend/tasks/offline_payload_retention.py b/products/ai_observability/backend/tasks/offline_payload_retention.pynew file mode 100644index 000000000000..bfdc5491d040--- /dev/null+++ b/products/ai_observability/backend/tasks/offline_payload_retention.py@@ -0,0 +1,46 @@+import time+from uuid import uuid4++import structlog+from celery import shared_task++from posthog.ph_client import ph_scoped_capture+from posthog.scoping_audit import skip_team_scope_audit+from posthog.utils import get_instance_region++from products.ai_observability.backend.offline_payload_retention import OfflinePayloadRetention++logger = structlog.get_logger(__name__)+++@shared_task(ignore_result=True, soft_time_limit=240, time_limit=300)+@skip_team_scope_audit+def expire_offline_evaluation_payloads() -> None:+ sweep = OfflinePayloadRetention()+ started = time.monotonic()+ status = "failed"+ try:+ sweep.run()+ status = "completed"+ finally:+ properties = {+ "run_id": str(uuid4()),+ "status": status,+ "batches": sweep.batches,+ "item_payloads_expired": sweep.item_payloads_expired,+ "result_payloads_expired": sweep.result_payloads_expired,+ "payloads_expired": sweep.item_payloads_expired + sweep.result_payloads_expired,+ "budget_exhausted": sweep.budget_exhausted,+ "duration_seconds": time.monotonic() - started,+ "$process_person_profile": False,+ }+ logger.info("offline_evaluation_payload_retention_run", **properties)+ try:+ with ph_scoped_capture(region=get_instance_region() or "US") as capture:+ capture(+ distinct_id="offline-evaluation-payload-retention",+ event="offline evaluation payload retention run",+ properties=properties,+ )+ except Exception:+ logger.exception("offline_evaluation_payload_retention_capture_failed")diff --git a/products/ai_observability/backend/test/test_offline_evaluation_reads.py b/products/ai_observability/backend/test/test_offline_evaluation_reads.pyindex 38d0aa365b86..ddcfb465b2f3 100644--- a/products/ai_observability/backend/test/test_offline_evaluation_reads.py+++ b/products/ai_observability/backend/test/test_offline_evaluation_reads.py@@ -35,6 +35,7 @@ from products.ai_observability.backend.offline_evaluation_service import (OfflineEvaluationValidationError,)from products.ai_observability.backend.offline_evaluation_types import JSONValue, ResultValue+from products.ai_observability.backend.offline_payload_retention import OfflinePayloadRetentionfrom products.ai_observability.backend.read_pagination import decode_cursor, encode_cursor@@ -570,20 +571,32 @@ class TestOfflineEvaluationReads(TestCase):item.save(update_fields=["accepted_at"])self._payload(item, {"input": None})self._payload(result, {})+ self.assertEqual(self.service.get_item_payload(experiment.id, item.id).payload_state, "available")self.assertEqual(self.service.get_item_payload(experiment.id, item.id).data, {"input": None})+ self.assertEqual(+ self.service.list_items(experiment.id, OfflineReadQuery()).results[0].payload_state, "available"+ )self.assertEqual(self.service.get_result_payload(experiment.id, result.id).data, {})before = self.service.list_summaries(experiment.id, OfflineReadQuery())- OfflineExperimentItemPayload.objects.for_team(self.team.id).filter(item_id=item.id).delete()- item.payload_state = PayloadState.EXPIRED- item.save(update_fields=["payload_state"])+ OfflinePayloadRetention().run()expired = self.service.get_item_payload(experiment.id, item.id)self.assertEqual((expired.payload_state, expired.available, expired.data), ("expired", False, None))+ self.assertEqual(self.service.list_items(experiment.id, OfflineReadQuery()).results[0].payload_state, "expired")self.assertTrue(self.service.get_result_payload(experiment.id, result.id).available)self.assertEqual(self.service.list_summaries(experiment.id, OfflineReadQuery()), before)self.assertEqual(self.service.scorer_history(self.numeric.definition_id, OfflineReadQuery()).results[0].summary,before.results[0],)+ result.accepted_at = item.accepted_at+ result.payload_expires_at = item.payload_expires_at+ result.save(update_fields=["accepted_at", "payload_expires_at"])+ self.assertEqual(self.service.get_result_payload(experiment.id, result.id).data, {})+ OfflinePayloadRetention().run()+ expired_result = self.service.get_result_payload(experiment.id, result.id)+ self.assertEqual(+ (expired_result.payload_state, expired_result.available, expired_result.data), ("expired", False, None)+ )def test_metadata_only_counts_and_nested_identity_checks(self) -> None:experiment, other_experiment = self._experiment(), self._experiment()diff --git a/products/ai_observability/backend/test/test_offline_evaluation_service.py b/products/ai_observability/backend/test/test_offline_evaluation_service.pyindex 1f5277f6f600..0981eb288e8d 100644--- a/products/ai_observability/backend/test/test_offline_evaluation_service.py+++ b/products/ai_observability/backend/test/test_offline_evaluation_service.py@@ -6,6 +6,9 @@ from threading import Eventfrom typing import Literalfrom uuid import uuid4+import time_machine+from unittest.mock import patch+from django.db import IntegrityError, connection, transactionfrom django.test import TestCase, TransactionTestCasefrom django.utils import timezone@@ -13,6 +16,7 @@ from django.utils import timezonefrom parameterized import parameterizedfrom posthog.models import Organization, Team+from posthog.models.scoping.manager import TeamScopedQuerySetfrom products.ai_observability.backend.models.datasets import Dataset, DatasetItem, DatasetItemVersion, DatasetRevisionfrom products.ai_observability.backend.models.offline_evaluations import (@@ -40,6 +44,8 @@ from products.ai_observability.backend.offline_evaluation_types import (ResultSubmission,UploadSubmission,)+from products.ai_observability.backend.offline_payload_retention import OfflinePayloadRetention+from products.ai_observability.backend.tasks.offline_payload_retention import expire_offline_evaluation_payloadsclass TestOfflineEvaluationService(TestCase):@@ -461,14 +467,8 @@ class TestOfflineEvaluationService(TestCase):accepted = self.ingestion.upload(self.experiment.id, self.submission)item = OfflineExperimentItem.objects.for_team(self.team.id).get(id=self.item.id)result = OfflineEvaluationResult.objects.for_team(self.team.id).get(id=accepted.results[0].id)- OfflineExperimentItemPayload.objects.for_team(self.team.id).filter(item=item).delete()- OfflineEvaluationResultPayload.objects.for_team(self.team.id).filter(result=result).delete()- OfflineExperimentItem.objects.for_team(self.team.id).filter(id=item.id).update(- payload_state=PayloadState.EXPIRED- )- OfflineEvaluationResult.objects.for_team(self.team.id).filter(id=result.id).update(- payload_state=PayloadState.EXPIRED- )+ with time_machine.travel(item.payload_expires_at, tick=False):+ OfflinePayloadRetention().run()replay = self.ingestion.upload(self.experiment.id, self.submission)@@ -481,6 +481,218 @@ class TestOfflineEvaluationService(TestCase):self.assertEqual(refreshed_result.payload_state, PayloadState.EXPIRED)self.assertFalse(OfflineExperimentItemPayload.objects.for_team(self.team.id).exists())self.assertFalse(OfflineEvaluationResultPayload.objects.for_team(self.team.id).exists())+ self.assertEqual(OfflineEvaluationUsageLedger.objects.for_team(self.team.id).count(), 1)++ @parameterized.expand([(-1, 0), (0, 2), (1, 2)])+ def test_retention_boundary_and_run_event(self, seconds: int, expected: int) -> None:+ receipt = self.ingestion.upload(self.experiment.id, self.submission)+ experiment_before = OfflineExperiment.objects.for_team(self.team.id).values().get()+ item_before = OfflineExperimentItem.objects.for_team(self.team.id).values().get()+ result_before = OfflineEvaluationResult.objects.for_team(self.team.id).values().get()+ ledger_before = OfflineEvaluationUsageLedger.objects.for_team(self.team.id).values().get()+ with (+ time_machine.travel(receipt.items[0].accepted_at + timedelta(days=30, seconds=seconds), tick=False),+ patch("products.ai_observability.backend.tasks.offline_payload_retention.ph_scoped_capture") as scoped,+ ):+ expire_offline_evaluation_payloads()+ properties = scoped.return_value.__enter__.return_value.call_args.kwargs["properties"]+ self.assertEqual(properties["payloads_expired"], expected)+ self.assertEqual(properties["batches"], expected)+ self.assertEqual(properties["status"], "completed")+ self.assertEqual(OfflineExperimentItemPayload.objects.for_team(self.team.id).count(), 1 - expected // 2)+ self.assertEqual(OfflineEvaluationResultPayload.objects.for_team(self.team.id).count(), 1 - expected // 2)+ expire_offline_evaluation_payloads()+ self.assertEqual(+ scoped.return_value.__enter__.return_value.call_args.kwargs["properties"]["payloads_expired"], 0+ )+ if expected:+ item_before["payload_state"] = PayloadState.EXPIRED+ result_before["payload_state"] = PayloadState.EXPIRED+ self.assertEqual(OfflineExperiment.objects.for_team(self.team.id).values().get(), experiment_before)+ self.assertEqual(OfflineExperimentItem.objects.for_team(self.team.id).values().get(), item_before)+ self.assertEqual(OfflineEvaluationResult.objects.for_team(self.team.id).values().get(), result_before)+ self.assertEqual(OfflineEvaluationUsageLedger.objects.for_team(self.team.id).values().get(), ledger_before)++ def test_retention_preserves_recent_payloads_and_items_without_payloads(self) -> None:+ now = timezone.now()+ old_item = replace(self.item, id=uuid4())+ absent_item = replace(self.item, id=uuid4(), payload=None)+ with time_machine.travel(now - timedelta(days=31), tick=False):+ self.ingestion.upload(+ self.experiment.id,+ UploadSubmission(+ items=[old_item, absent_item],+ results=[+ replace(self.result, item_id=old_item.id),+ replace(self.result, item_id=absent_item.id, payload=None),+ ],+ ),+ )+ self.ingestion.upload(self.experiment.id, self.submission)++ sweep = OfflinePayloadRetention()+ sweep.run()++ self.assertEqual((sweep.item_payloads_expired, sweep.result_payloads_expired), (1, 1))+ self.assertEqual(+ OfflineExperimentItemPayload.objects.for_team(self.team.id).values_list("item_id", "data").get(),+ (self.item.id, self.item.payload),+ )+ self.assertEqual(+ OfflineEvaluationResultPayload.objects.for_team(self.team.id).values_list("result__item_id", "data").get(),+ (self.item.id, self.result.payload),+ )+ self.assertEqual(+ dict(OfflineExperimentItem.objects.for_team(self.team.id).values_list("id", "payload_state")),+ {old_item.id: "expired", absent_item.id: "not_provided", self.item.id: "available"},+ )+ self.assertEqual(+ dict(OfflineEvaluationResult.objects.for_team(self.team.id).values_list("item_id", "payload_state")),+ {old_item.id: "expired", absent_item.id: "not_provided", self.item.id: "available"},+ )++ @parameterized.expand([("item",), ("result",)])+ def test_retention_rolls_back_payload_deletion_and_reports_failure(self, failing_owner: str) -> None:+ receipt = self.ingestion.upload(self.experiment.id, self.submission)+ failing_model = OfflineExperimentItem if failing_owner == "item" else OfflineEvaluationResult+ original_update = TeamScopedQuerySet.update++ def fail_state_update(+ queryset: TeamScopedQuerySet[OfflineExperimentItem | OfflineEvaluationResult], **kwargs: object+ ) -> int:+ if queryset.model is failing_model:+ raise RuntimeError("write failed")+ return original_update(queryset, **kwargs)++ with (+ time_machine.travel(receipt.items[0].accepted_at + timedelta(days=30), tick=False),+ patch("products.ai_observability.backend.tasks.offline_payload_retention.ph_scoped_capture") as scoped,+ ):+ with (+ patch.object(TeamScopedQuerySet, "update", autospec=True, side_effect=fail_state_update),+ self.assertRaisesMessage(RuntimeError, "write failed"),+ ):+ expire_offline_evaluation_payloads()+ properties = scoped.return_value.__enter__.return_value.call_args.kwargs["properties"]+ self.assertEqual(properties["status"], "failed")+ committed = int(failing_owner == "result")+ self.assertEqual(properties["batches"], committed)+ self.assertEqual(properties["payloads_expired"], committed)+ self.assertEqual(OfflineExperimentItemPayload.objects.for_team(self.team.id).exists(), not committed)+ self.assertTrue(OfflineEvaluationResultPayload.objects.for_team(self.team.id).exists())+ self.assertEqual(failing_model.objects.for_team(self.team.id).get().payload_state, PayloadState.AVAILABLE)+ expire_offline_evaluation_payloads()+ retry = scoped.return_value.__enter__.return_value.call_args.kwargs["properties"]+ self.assertEqual(retry["status"], "completed")+ self.assertEqual(retry["payloads_expired"], 2 - committed)+ self.assertFalse(OfflineExperimentItemPayload.objects.for_team(self.team.id).exists())+ self.assertFalse(OfflineEvaluationResultPayload.objects.for_team(self.team.id).exists())++ @parameterized.expand([([0, 180], 0), ([0, 0, 0, 180], 1)])+ def test_retention_time_budget_leaves_remaining_payloads(self, clock: list[int], expired_items: int) -> None:+ receipt = self.ingestion.upload(self.experiment.id, self.submission)+ with (+ time_machine.travel(receipt.items[0].accepted_at + timedelta(days=30), tick=False),+ patch("products.ai_observability.backend.offline_payload_retention.time") as mocked_time,+ ):+ mocked_time.monotonic.side_effect = clock+ sweep = OfflinePayloadRetention()+ sweep.run()+ self.assertTrue(sweep.budget_exhausted)+ self.assertEqual((sweep.item_payloads_expired, sweep.result_payloads_expired), (expired_items, 0))+ self.assertEqual(OfflineExperimentItemPayload.objects.for_team(self.team.id).count(), 1 - expired_items)+ self.assertEqual(OfflineEvaluationResultPayload.objects.for_team(self.team.id).count(), 1)++ def test_retention_batch_limit_leaves_work_for_next_run(self) -> None:+ for _ in range(2):+ item = replace(self.item, id=uuid4())+ self.ingestion.upload(+ self.experiment.id, UploadSubmission(items=[item], results=[replace(self.result, item_id=item.id)])+ )+ with (+ time_machine.travel(timezone.now() + timedelta(days=31), tick=False),+ patch("products.ai_observability.backend.offline_payload_retention.BATCH_SIZE", 1),+ patch("products.ai_observability.backend.offline_payload_retention.MAX_BATCHES_PER_TYPE", 1),+ ):+ sweep = OfflinePayloadRetention()+ sweep.run()+ self.assertTrue(sweep.budget_exhausted)+ self.assertEqual((sweep.item_payloads_expired, sweep.result_payloads_expired), (1, 1))+ self.assertEqual(OfflineExperimentItemPayload.objects.for_team(self.team.id).count(), 1)+ OfflinePayloadRetention().run()+ self.assertFalse(OfflineExperimentItemPayload.objects.for_team(self.team.id).exists())++ def test_retention_checks_time_budget_between_environments(self) -> None:+ now = timezone.now()+ with time_machine.travel(now - timedelta(days=32), tick=False):+ self.ingestion.upload(self.experiment.id, self.submission)+ other_experiment = (+ OfflineExperimentService(team_id=self.other_team.id, user_access_control=None)+ .create(replace(self.experiment_submission, id=uuid4()))+ .experiment+ )+ item = replace(self.item, id=uuid4())+ with time_machine.travel(now - timedelta(days=31), tick=False):+ OfflineEvaluationIngestionService(team_id=self.other_team.id, user_access_control=None).upload(+ other_experiment.id,+ UploadSubmission(+ items=[item],+ results=[replace(self.result, item_id=item.id, scorer_version_id=self.other_version.id)],+ ),+ )+ sweep = OfflinePayloadRetention()+ with patch("products.ai_observability.backend.offline_payload_retention.time") as mocked_time:+ mocked_time.monotonic.side_effect = lambda: 180 if sweep.batches else 0+ sweep.run()++ self.assertTrue(sweep.budget_exhausted)+ self.assertEqual((sweep.batches, sweep.item_payloads_expired, sweep.result_payloads_expired), (1, 1, 0))+ self.assertFalse(OfflineExperimentItemPayload.objects.for_team(self.team.id).exists())+ self.assertTrue(OfflineExperimentItemPayload.objects.for_team(self.other_team.id).exists())+ self.assertEqual(+ OfflineExperimentItem.objects.for_team(self.other_team.id).get().payload_state, PayloadState.AVAILABLE+ )+ for team_id in (self.team.id, self.other_team.id):+ self.assertTrue(OfflineEvaluationResultPayload.objects.for_team(team_id).exists())++ retry = OfflinePayloadRetention()+ retry.run()+ self.assertEqual((retry.batches, retry.item_payloads_expired, retry.result_payloads_expired), (3, 1, 2))+ for team_id in (self.team.id, self.other_team.id):+ self.assertFalse(OfflineExperimentItemPayload.objects.for_team(team_id).exists())+ self.assertFalse(OfflineEvaluationResultPayload.objects.for_team(team_id).exists())++ @parameterized.expand([(0,), (31,)])+ def test_retention_keeps_environment_scoping(self, child_age_days: int) -> None:+ now = timezone.now()+ with time_machine.travel(now - timedelta(days=31), tick=False):+ self.ingestion.upload(self.experiment.id, self.submission)+ child = Team.objects.create(organization=self.team.organization, parent_team=self.team)+ scorer = ScoreDefinition.objects.create(team=child, name="Child accuracy", kind="numeric")+ version = scorer.create_new_version(config={}, created_by=None)+ child_experiment = (+ OfflineExperimentService(team_id=child.id, user_access_control=None)+ .create(replace(self.experiment_submission, id=uuid4()))+ .experiment+ )+ item = replace(self.item, id=uuid4())+ with time_machine.travel(now - timedelta(days=child_age_days), tick=False):+ OfflineEvaluationIngestionService(team_id=child.id, user_access_control=None).upload(+ child_experiment.id,+ UploadSubmission(+ items=[item], results=[replace(self.result, item_id=item.id, scorer_version_id=version.id)]+ ),+ )+ sweep = OfflinePayloadRetention()+ sweep.run()+ expired = 1 + int(child_age_days == 31)+ self.assertEqual((sweep.item_payloads_expired, sweep.result_payloads_expired), (expired, expired))+ self.assertEqual(sweep.batches, 2 * expired)+ for team_id in (self.team.id, child.id):+ retained = team_id == child.id and child_age_days == 0+ self.assertEqual(OfflineExperimentItemPayload.objects.for_team(team_id).exists(), retained)+ self.assertEqual(OfflineEvaluationResultPayload.objects.for_team(team_id).exists(), retained)+ self.assertEqual(OfflineEvaluationResult.objects.for_team(team_id).count(), 1)def test_previous_archived_version_uses_its_own_configuration(self) -> None:self.scorer.create_new_version(config={"min": 10, "max": 20}, created_by=None)@@ -585,6 +797,55 @@ class TestOfflineEvaluationServiceConcurrency(TransactionTestCase):results=[ResultSubmission(item_id=item.id, scorer_version_id=version.id, status="ok", value=0.8)],)+ def _run_retention(self) -> OfflinePayloadRetention:+ try:+ with connection.cursor() as cursor:+ cursor.execute("SET lock_timeout = '2s'")+ sweep = OfflinePayloadRetention()+ sweep.run()+ return sweep+ finally:+ connection.close()++ @parameterized.expand([("item",), ("result",)])+ def test_retention_skips_locked_payload_owners_and_recovers_on_next_run(self, locked_type: str) -> None:+ with time_machine.travel(timezone.now() - timedelta(days=31), tick=False):+ for _ in range(2):+ item = replace(self.submission.items[0], id=uuid4(), payload={"input": "Example"})+ self.ingestion.upload(+ self.experiment.id,+ UploadSubmission(+ items=[item],+ results=[+ replace(self.submission.results[0], item_id=item.id, payload={"reasoning": "Example"})+ ],+ ),+ )+ owner_model = OfflineExperimentItem if locked_type == "item" else OfflineEvaluationResult+ payload_model = OfflineExperimentItemPayload if locked_type == "item" else OfflineEvaluationResultPayload+ with ThreadPoolExecutor(max_workers=1) as executor:+ with transaction.atomic():+ locked = owner_model.objects.for_team(self.team.id).select_for_update().order_by("id").first()+ assert locked is not None+ sweep = executor.submit(self._run_retention).result(timeout=10)+ self.assertEqual(+ (sweep.item_payloads_expired, sweep.result_payloads_expired),+ (1, 2) if locked_type == "item" else (2, 1),+ )+ self.assertTrue(payload_model.objects.for_team(self.team.id).filter(pk=locked.pk).exists())+ locked.refresh_from_db()+ self.assertEqual(locked.payload_state, PayloadState.AVAILABLE)+ retry = executor.submit(self._run_retention).result(timeout=10)+ self.assertEqual(+ (retry.item_payloads_expired, retry.result_payloads_expired),+ (1, 0) if locked_type == "item" else (0, 1),+ )+ self.assertFalse(OfflineExperimentItemPayload.objects.for_team(self.team.id).exists())+ self.assertFalse(OfflineEvaluationResultPayload.objects.for_team(self.team.id).exists())+ self.assertEqual(OfflineExperimentItem.objects.for_team(self.team.id).count(), 2)+ self.assertEqual(OfflineEvaluationResult.objects.for_team(self.team.id).count(), 2)+ self.assertEqual(OfflineEvaluationUsageLedger.objects.for_team(self.team.id).count(), 2)+def _run_second_operation(self, operation: Callable[[], UploadReceipt | ExperimentReceipt], started: Event) -> UploadReceipt | ExperimentReceipt:diff --git a/products/ai_observability/frontend/generated/api.schemas.ts b/products/ai_observability/frontend/generated/api.schemas.tsindex 80fe61661fc4..43f3cd85ad41 100644--- a/products/ai_observability/frontend/generated/api.schemas.ts+++ b/products/ai_observability/frontend/generated/api.schemas.ts@@ -729,7 +729,7 @@ export interface OfflineResultCellApi {* * `expired` - Expired */payload_state: PayloadStateEnumApi/**- * Payload retention deadline; cleanup is not yet enabled.+ * Payload expires 30 days after first acceptance; deletion runs hourly.* @nullable*/payload_expires_at: string | null@@ -786,7 +786,7 @@ export interface OfflineItemReadApi {* * `expired` - Expired */payload_state: PayloadStateEnumApi/**- * Payload retention deadline; cleanup is not yet enabled.+ * Payload expires 30 days after first acceptance; deletion runs hourly.* @nullable*/payload_expires_at: string | null@@ -1056,7 +1056,7 @@ export interface OfflineResultReadApi {* * `expired` - Expired */payload_state: PayloadStateEnumApi/**- * Payload retention deadline; cleanup is not yet enabled.+ * Payload expires 30 days after first acceptance; deletion runs hourly.* @nullable*/payload_expires_at: string | nulldiff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 63c7fdbbdc92..38899f78d7de 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -75610,7 +75610,7 @@ export namespace Schemas {* * `expired` - Expired */payload_state: PayloadStateEnum;/**- * Payload retention deadline; cleanup is not yet enabled.+ * Payload expires 30 days after first acceptance; deletion runs hourly.* @nullable*/payload_expires_at: string | null;@@ -75667,7 +75667,7 @@ export namespace Schemas {* * `expired` - Expired */payload_state: PayloadStateEnum;/**- * Payload retention deadline; cleanup is not yet enabled.+ * Payload expires 30 days after first acceptance; deletion runs hourly.* @nullable*/payload_expires_at: string | null;@@ -75754,7 +75754,7 @@ export namespace Schemas {* * `expired` - Expired */payload_state: PayloadStateEnum;/**- * Payload retention deadline; cleanup is not yet enabled.+ * Payload expires 30 days after first acceptance; deletion runs hourly.* @nullable*/payload_expires_at: string | null;