:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(ai-training): shred v4 replay keys on deletion (#121247)
Nicholas Waltz37565421b3e6
diff --git a/posthog/settings/ingestion.py b/posthog/settings/ingestion.pyindex 22d873dce765..3984a01c444d 100644--- a/posthog/settings/ingestion.py+++ b/posthog/settings/ingestion.py@@ -160,3 +160,6 @@ AI_RESEARCH_REPLAY_KEY_TABLE = os.getenv("AI_RESEARCH_REPLAY_KEY_TABLE", "")AI_RESEARCH_REPLAY_AWS_REGION = os.getenv("AI_RESEARCH_REPLAY_AWS_REGION", "us-east-1")AI_RESEARCH_REPLAY_DYNAMODB_ENDPOINT = os.getenv("AI_RESEARCH_REPLAY_DYNAMODB_ENDPOINT", "")AI_RESEARCH_REPLAY_KMS_KEY_ARN = os.getenv("AI_RESEARCH_REPLAY_KMS_KEY_ARN", "")+AI_RESEARCH_V4_KEY_TABLE = os.getenv("AI_RESEARCH_V4_KEY_TABLE", "")+AI_RESEARCH_V4_AWS_REGION = os.getenv("AI_RESEARCH_V4_AWS_REGION", "us-east-1")+AI_RESEARCH_V4_DYNAMODB_ENDPOINT = os.getenv("AI_RESEARCH_V4_DYNAMODB_ENDPOINT", "")diff --git a/products/ai_training/backend/config.py b/products/ai_training/backend/config.pyindex 9a4918b38a13..a113c78377c0 100644--- a/products/ai_training/backend/config.py+++ b/products/ai_training/backend/config.py@@ -5,5 +5,9 @@ def key_table_name() -> str:return str(getattr(settings, "AI_RESEARCH_REPLAY_KEY_TABLE", ""))+def v4_key_table_name() -> str:+ return str(getattr(settings, "AI_RESEARCH_V4_KEY_TABLE", ""))++def privacy_enabled() -> bool:- return bool(key_table_name())+ return bool(key_table_name() or v4_key_table_name())diff --git a/products/ai_training/backend/management/commands/delete_ai_training_month.py b/products/ai_training/backend/management/commands/delete_ai_training_month.pyindex baa59cf5e02e..eef1c6033f17 100644--- a/products/ai_training/backend/management/commands/delete_ai_training_month.py+++ b/products/ai_training/backend/management/commands/delete_ai_training_month.py@@ -4,7 +4,7 @@ from typing import Anyfrom django.core.management.base import BaseCommand, CommandErrorfrom products.ai_training.backend.config import key_table_name-from products.ai_training.backend.privacy.store import AITrainingPrivacyStore+from products.ai_training.backend.privacy.store import V3KeyTargetclass Command(BaseCommand):@@ -17,7 +17,7 @@ class Command(BaseCommand):if not key_table_name():raise CommandError("AI_RESEARCH_REPLAY_KEY_TABLE is not configured")try:- count = AITrainingPrivacyStore.from_settings().delete_month(options["session_month"])+ count = V3KeyTarget.from_settings().delete_month(options["session_month"])except ValueError as error:raise CommandError(str(error)) from errorself.stdout.write(f"Removed {count} indexed keys. A reader that already cached a key can still use it.")diff --git a/products/ai_training/backend/privacy/keys.py b/products/ai_training/backend/privacy/keys.pynew file mode 100644index 000000000000..460efbaeb16c--- /dev/null+++ b/products/ai_training/backend/privacy/keys.py@@ -0,0 +1,92 @@+import hashlib+from datetime import UTC, datetime, timedelta+from typing import Protocol, TypedDict, cast++import boto3+from botocore.config import Config++from posthog.dataclasses import frozen++from products.ai_training.backend.models import AITrainingDeletionRequest++KEY_SHARDS = 32+# A per-session deletion keeps the team month key, so it must also remove the seal that the month key opens.+KEY_MATERIAL_ATTRIBUTES = ("wrapped_key", "sealed_key", "key_nonce")+DynamoItem = dict[str, dict[str, str | bool | bytes]]+++class DynamoResponse(TypedDict, total=False):+ Item: DynamoItem+ Items: list[DynamoItem]+ LastEvaluatedKey: DynamoItem+++class DeletionWork(TypedDict, total=False):+ op: str+ team_id: int+ shard: int+ after: DynamoItem+++class PrivacyDynamoClient(Protocol):+ def put_item(self, **kwargs: object) -> DynamoResponse: ...+ def get_item(self, **kwargs: object) -> DynamoResponse: ...+ def query(self, **kwargs: object) -> DynamoResponse: ...+ def transact_write_items(self, **kwargs: object) -> DynamoResponse: ...+++class DeletionTarget(Protocol):+ # The request cursor stores each target's work under this key, so targets progress independently.+ cursor_key: str+ table_name: str++ def initialize(self, request: AITrainingDeletionRequest) -> list[DeletionWork]: ...+ def advance(self, work: DeletionWork) -> list[DeletionWork]: ...+++def dynamo_client(region: str, endpoint: str) -> PrivacyDynamoClient:+ client = boto3.client(+ "dynamodb",+ region_name=region,+ endpoint_url=endpoint or None,+ config=Config(connect_timeout=5, read_timeout=5, retries={"max_attempts": 4, "mode": "standard"}),+ )+ return cast(PrivacyDynamoClient, client)+++def item_key(pk: str, sk: str) -> DynamoItem:+ return {"pk": {"S": pk}, "sk": {"S": sk}}+++def month_end(session_month: str) -> datetime:+ year, month = (int(part) for part in session_month.split("-"))+ return datetime(year + month // 12, month % 12 + 1, 1, tzinfo=UTC)+++def identity_digest(value: str) -> str:+ return hashlib.sha256(value.encode()).hexdigest()+++def session_key(team_id: int, session_id: str) -> DynamoItem:+ shard = int(identity_digest(session_id)[:8], 16) % KEY_SHARDS+ return item_key(f"team:{team_id}:shard:{shard}", f"session:{session_id}")+++@frozen+class ReachableMonths:+ previous: str+ current: str+ following: str++ @property+ def months(self) -> list[str]:+ return [self.previous, self.current, self.following]+++def reachable_months(now: datetime) -> ReachableMonths:+ current = f"{now:%Y-%m}"+ return ReachableMonths(+ previous=f"{datetime(now.year, now.month, 1, tzinfo=UTC) - timedelta(days=1):%Y-%m}",+ current=current,+ following=f"{month_end(current):%Y-%m}",+ )diff --git a/products/ai_training/backend/privacy/store.py b/products/ai_training/backend/privacy/store.pyindex 3766b030f439..85be122b2710 100644--- a/products/ai_training/backend/privacy/store.py+++ b/products/ai_training/backend/privacy/store.py@@ -1,85 +1,51 @@import reimport time-import hashlibfrom collections.abc import Sequence-from datetime import UTC, datetime, timedelta-from typing import Protocol, TypedDict, cast+from datetime import timedeltafrom django.conf import settingsfrom django.db import transactionfrom django.utils import timezone-import boto3import structlog-from botocore.config import Config-from products.ai_training.backend.config import key_table_name+from products.ai_training.backend.config import key_table_name, v4_key_table_namefrom products.ai_training.backend.models import AITrainingDeletionRequest+from products.ai_training.backend.privacy.keys import (+ KEY_MATERIAL_ATTRIBUTES,+ KEY_SHARDS,+ DeletionTarget,+ DeletionWork,+ DynamoItem,+ DynamoResponse,+ PrivacyDynamoClient,+ dynamo_client,+ item_key,+ month_end,+ reachable_months,+ session_key,+)+from products.ai_training.backend.privacy.v4 import V4KeyTargetlogger = structlog.get_logger(__name__)-KEY_SHARDS = 32# Equals ML_SESSION_MAX_AGE_DAYS in nodejs/src/ingestion/pipelines/sessionreplay/ml-mirror/session-identifier-format.ts: ingestion drops sessions that started earlier than that, so no key for a month can appear after the month end plus this period.MONTH_DELETE_GRACE_DAYS = 14# A batch admitted just inside the grace period still commits within its 45 s budget, so deletion stays behind that too.MONTH_DELETE_IN_FLIGHT_MARGIN = timedelta(hours=1)-# A per-session deletion keeps the team month key, so it must also remove the seal that the month key opens.-KEY_MATERIAL_ATTRIBUTES = ("wrapped_key", "sealed_key", "key_nonce")-DynamoItem = dict[str, dict[str, str | bool | bytes]]-class DynamoResponse(TypedDict, total=False):- Item: DynamoItem- Items: list[DynamoItem]- LastEvaluatedKey: DynamoItem+class V3KeyTarget:+ cursor_key = "work"--class DeletionWork(TypedDict, total=False):- op: str- team_id: int- shard: int- after: DynamoItem---class PrivacyDynamoClient(Protocol):- def put_item(self, **kwargs: object) -> DynamoResponse: ...- def get_item(self, **kwargs: object) -> DynamoResponse: ...- def query(self, **kwargs: object) -> DynamoResponse: ...- def transact_write_items(self, **kwargs: object) -> DynamoResponse: ...---def item_key(pk: str, sk: str) -> DynamoItem:- return {"pk": {"S": pk}, "sk": {"S": sk}}---def month_end(session_month: str) -> datetime:- year, month = (int(part) for part in session_month.split("-"))- return datetime(year + month // 12, month % 12 + 1, 1, tzinfo=UTC)---def identity_digest(value: str) -> str:- return hashlib.sha256(value.encode()).hexdigest()---def session_key(team_id: int, session_id: str) -> DynamoItem:- shard = int(identity_digest(session_id)[:8], 16) % KEY_SHARDS- return item_key(f"team:{team_id}:shard:{shard}", f"session:{session_id}")---class AITrainingPrivacyStore:def __init__(self, client: PrivacyDynamoClient, table_name: str) -> None:self.client = clientself.table_name = table_name@classmethod- def from_settings(cls) -> "AITrainingPrivacyStore":- client = boto3.client(- "dynamodb",- region_name=settings.AI_RESEARCH_REPLAY_AWS_REGION,- endpoint_url=settings.AI_RESEARCH_REPLAY_DYNAMODB_ENDPOINT or None,- config=Config(connect_timeout=5, read_timeout=5, retries={"max_attempts": 4, "mode": "standard"}),- )- return cls(cast(PrivacyDynamoClient, client), key_table_name())+ def from_settings(cls) -> "V3KeyTarget":+ client = dynamo_client(settings.AI_RESEARCH_REPLAY_AWS_REGION, settings.AI_RESEARCH_REPLAY_DYNAMODB_ENDPOINT)+ return cls(client, key_table_name())def delete_month(self, session_month: str) -> int:if re.fullmatch(r"[0-9]{4}-(0[1-9]|1[0-2])", session_month) is None:@@ -138,11 +104,7 @@ class AITrainingPrivacyStore:# A sweep reaches only an image key that already exists, so a team that keeps sending would open a month the sweep never saw. Ingestion admits a session up to ML_SESSION_MAX_AGE_DAYS old, which is inside one month, so these three are every month a later session can still open. An update creates the row, so each one closes whether or not it holds a key.def shred_reachable_months(self, team_id: int) -> None:- now = timezone.now()- current = f"{now:%Y-%m}"- previous = f"{datetime(now.year, now.month, 1, tzinfo=UTC) - timedelta(days=1):%Y-%m}"- months = (previous, current, f"{month_end(current):%Y-%m}")- self.shred([item_key(f"team:{team_id}", f"image:{month}") for month in months])+ self.shred([item_key(f"team:{team_id}", f"image:{month}") for month in reachable_months(timezone.now()).months])def initialize(self, request: AITrainingDeletionRequest) -> list[DeletionWork]:if request.team_id is None:@@ -173,16 +135,34 @@ class AITrainingPrivacyStore:return [next_work]return []++class AITrainingPrivacyStore:+ def __init__(self, targets: Sequence[DeletionTarget]) -> None:+ self.targets = list(targets)++ @classmethod+ def from_settings(cls) -> "AITrainingPrivacyStore":+ targets: list[DeletionTarget] = []+ if key_table_name():+ targets.append(V3KeyTarget.from_settings())+ if v4_key_table_name():+ targets.append(V4KeyTarget.from_settings())+ return cls(targets)+def apply(self, request: AITrainingDeletionRequest, deadline: float) -> bool:- work = request.cursor.get("work")- if work is None:- work = self.initialize(request)- self.save_cursor(request, work=work)- while work:- if time.monotonic() >= deadline:- return False- work = self.advance(work[0]) + work[1:]- self.save_cursor(request, work=work)+ for target in self.targets:+ work = request.cursor.get(target.cursor_key)+ if work is None:+ # Initialization writes with retries, so a target that starts after the deadline can overrun the task time limit.+ if time.monotonic() >= deadline:+ return False+ work = target.initialize(request)+ self.save_cursor(request, **{target.cursor_key: work})+ while work:+ if time.monotonic() >= deadline:+ return False+ work = target.advance(work[0]) + work[1:]+ self.save_cursor(request, **{target.cursor_key: work})# A key that a batch stores after the sweep passes its shard is sealed under a month key this deletion already tombstoned, so a second sweep finds nothing that is readable. The sweep removes rows to save cost. See products/ai_training/docs/replay-data.md.request.completed_at = timezone.now()request.identifiers = []diff --git a/products/ai_training/backend/privacy/v4.py b/products/ai_training/backend/privacy/v4.pynew file mode 100644index 000000000000..c0f7fa820f5c--- /dev/null+++ b/products/ai_training/backend/privacy/v4.py@@ -0,0 +1,120 @@+from collections.abc import Sequence+from datetime import UTC, datetime++from django.conf import settings+from django.utils import timezone++from products.ai_training.backend.config import v4_key_table_name+from products.ai_training.backend.models import AITrainingDeletionRequest+from products.ai_training.backend.privacy.keys import (+ KEY_MATERIAL_ATTRIBUTES,+ KEY_SHARDS,+ DeletionWork,+ DynamoItem,+ DynamoResponse,+ PrivacyDynamoClient,+ dynamo_client,+ item_key,+ reachable_months,+ session_key,+)++# Equals the expiry rule of the v4 recorder: a row lives until the start of the fifth month after its key month.+KEY_RETENTION_MONTHS = 5+++def key_expiry(key_month: str) -> int:+ year, month = (int(part) for part in key_month.split("-"))+ index = year * 12 + month - 1 + KEY_RETENTION_MONTHS+ return int(datetime(index // 12, index % 12 + 1, 1, tzinfo=UTC).timestamp())+++class V4KeyTarget:+ cursor_key = "v4_work"++ def __init__(self, client: PrivacyDynamoClient, table_name: str) -> None:+ self.client = client+ self.table_name = table_name++ @classmethod+ def from_settings(cls) -> "V4KeyTarget":+ client = dynamo_client(settings.AI_RESEARCH_V4_AWS_REGION, settings.AI_RESEARCH_V4_DYNAMODB_ENDPOINT)+ return cls(client, v4_key_table_name())++ def page(self, pk: str, prefix: str, cursor: DynamoItem | None = None) -> DynamoResponse:+ return self.client.query(+ TableName=self.table_name,+ KeyConditionExpression="pk = :pk AND begins_with(sk, :prefix)",+ ExpressionAttributeValues={":pk": {"S": pk}, ":prefix": {"S": prefix}},+ ProjectionExpression="pk, sk",+ ConsistentRead=True,+ Limit=50,+ **({"ExclusiveStartKey": cursor} if cursor else {}),+ )++ # An update creates a row that does not exist yet, and writers never overwrite a row, so the tombstone needs an expiry or it stays forever. if_not_exists keeps the expiry of a row the recorder wrote.+ def tombstone(self, rows: Sequence[tuple[DynamoItem, int]]) -> None:+ unique = {str(key["pk"]["S"]) + "\0" + str(key["sk"]["S"]): (key, expiry) for key, expiry in rows}+ values = list(unique.values())+ for offset in range(0, len(values), 100):+ self.client.transact_write_items(+ TransactItems=[+ {+ "Update": {+ "TableName": self.table_name,+ "Key": {"pk": key["pk"], "sk": key["sk"]},+ "UpdateExpression": "SET deleted = :true, expires_at = if_not_exists(expires_at, :expiry) REMOVE "+ + ", ".join(KEY_MATERIAL_ATTRIBUTES),+ "ExpressionAttributeValues": {":true": {"BOOL": True}, ":expiry": {"N": str(expiry)}},+ },+ }+ for key, expiry in values[offset : offset + 100]+ ]+ )++ def month_row(self, team_id: int, key_month: str) -> tuple[DynamoItem, int]:+ return item_key(f"team:{team_id}", f"month:{key_month}"), key_expiry(key_month)++ def session_expiry(self) -> int:+ return key_expiry(f"{timezone.now():%Y-%m}")++ def initialize(self, request: AITrainingDeletionRequest) -> list[DeletionWork]:+ if request.team_id is None:+ raise ValueError("AI training deletion request has no team")+ team_id = request.team_id+ if request.kind == "team":+ # A session can still open these months after the sweep passes, and a tombstone stops the recorder from writing a key there.+ self.tombstone([self.month_row(team_id, month) for month in reachable_months(timezone.now()).months])+ return [{"op": "team", "team_id": team_id, "shard": -1}]+ if request.kind == "session":+ expiry = self.session_expiry()+ self.tombstone([(session_key(team_id, value), expiry) for value in request.identifiers])+ return []+ raise ValueError("Unknown AI training deletion request")++ def advance(self, work: DeletionWork) -> list[DeletionWork]:+ if work["op"] != "team":+ raise ValueError("Unknown AI training deletion operation")+ team_id = int(work["team_id"])+ shard = int(work["shard"])+ if shard == -1:+ response = self.page(f"team:{team_id}", "month:", work.get("after"))+ self.tombstone(+ [+ self.month_row(team_id, str(row["sk"]["S"]).removeprefix("month:"))+ for row in response.get("Items", [])+ ]+ )+ else:+ response = self.page(f"team:{team_id}:shard:{shard}", "session:", work.get("after"))+ expiry = self.session_expiry()+ self.tombstone(+ [(item_key(str(row["pk"]["S"]), str(row["sk"]["S"])), expiry) for row in response.get("Items", [])]+ )+ if response.get("LastEvaluatedKey"):+ return [{**work, "after": response["LastEvaluatedKey"]}]+ if shard + 1 < KEY_SHARDS:+ next_work: DeletionWork = {**work, "shard": shard + 1}+ next_work.pop("after", None)+ return [next_work]+ return []diff --git a/products/ai_training/backend/tests/test_privacy_store.py b/products/ai_training/backend/tests/test_privacy_store.pyindex 3adcb96f316b..75216d48dd45 100644--- a/products/ai_training/backend/tests/test_privacy_store.py+++ b/products/ai_training/backend/tests/test_privacy_store.py@@ -1,5 +1,6 @@import timefrom datetime import datetime+from typing import Anyfrom unittest.mock import MagicMock, patch@@ -7,17 +8,28 @@ from django.test import SimpleTestCasefrom parameterized import parameterized-from products.ai_training.backend.privacy.store import AITrainingPrivacyStore, DynamoResponse, item_key, session_key+from products.ai_training.backend.privacy.keys import DynamoResponse, item_key, session_key+from products.ai_training.backend.privacy.store import AITrainingPrivacyStore, V3KeyTarget+from products.ai_training.backend.privacy.v4 import V4KeyTargetfrom products.ai_training.backend.tasks.tasks import process_ai_training_privacy_requests+NOW = "products.ai_training.backend.privacy.store.timezone.now"+TargetClass = type[V3KeyTarget] | type[V4KeyTarget]+++def epoch(day: str) -> str:+ return str(int(datetime.fromisoformat(f"{day}T00:00:00+00:00").timestamp()))+class TestAITrainingPrivacyStore(SimpleTestCase):- @parameterized.expand([("disabled", "", False), ("enabled", "test-table", True)])+ @parameterized.expand(+ [("disabled", "", "", False), ("v3", "test-table", "", True), ("v4", "", "test-v4-table", True)]+ )def test_privacy_task_uses_dedicated_queue_and_only_drains_when_enabled(- self, _name: str, table: str, enabled: bool+ self, _name: str, table: str, v4_table: str, enabled: bool) -> None:with (- self.settings(AI_RESEARCH_REPLAY_KEY_TABLE=table),+ self.settings(AI_RESEARCH_REPLAY_KEY_TABLE=table, AI_RESEARCH_V4_KEY_TABLE=v4_table),patch("products.ai_training.backend.tasks.tasks.AITrainingPrivacyStore.from_settings") as store,):signature = process_ai_training_privacy_requests.signature()@@ -46,7 +58,7 @@ class TestAITrainingPrivacyStore(SimpleTestCase):return pages.pop(0)client.query.side_effect = query- store = AITrainingPrivacyStore(client, "table")+ store = V3KeyTarget(client, "table")self.assertEqual(store.delete_month("2025-09"), 2)client.put_item.assert_not_called()self.assertEqual(@@ -74,54 +86,91 @@ class TestAITrainingPrivacyStore(SimpleTestCase):) -> None:client = MagicMock()client.query.return_value = {"Items": []}- store = AITrainingPrivacyStore(client, "table")- with patch("products.ai_training.backend.privacy.store.timezone.now", return_value=datetime.fromisoformat(now)):+ store = V3KeyTarget(client, "table")+ with patch(NOW, return_value=datetime.fromisoformat(now)):if allowed:self.assertEqual(store.delete_month(month), 0)else:with self.assertRaisesRegex(ValueError, "can be deleted from"):store.delete_month(month)- def test_session_deletion_shreds_every_key_attribute_without_querying_user_indexes(self) -> None:+ @parameterized.expand(+ [+ ("v3", V3KeyTarget, "SET deleted = :deleted REMOVE ", {":deleted": {"BOOL": True}}),+ (+ "v4",+ V4KeyTarget,+ "SET deleted = :true, expires_at = if_not_exists(expires_at, :expiry) REMOVE ",+ {":true": {"BOOL": True}, ":expiry": {"N": epoch("2027-03-01")}},+ ),+ ]+ )+ def test_session_deletion_shreds_every_key_attribute_without_querying_user_indexes(+ self, _name: str, target: TargetClass, prefix: str, values: dict[str, dict[str, object]]+ ) -> None:client = MagicMock()sessions = ["01a09f92-e780-7000-8000-000000000001", "01a09f92-e780-7000-8000-000000000002"]- store = AITrainingPrivacyStore(client, "table")- self.assertEqual(store.initialize(MagicMock(kind="session", team_id=7, identifiers=sessions)), [])+ store = target(client, "table")+ with patch(NOW, return_value=datetime.fromisoformat("2026-10-31T23:59:00+00:00")):+ self.assertEqual(store.initialize(MagicMock(kind="session", team_id=7, identifiers=sessions)), [])updates = client.transact_write_items.call_args.kwargs["TransactItems"]self.assertEqual([update["Update"]["Key"] for update in updates], [session_key(7, value) for value in sessions])for update in updates:expression = update["Update"]["UpdateExpression"]- self.assertTrue(expression.startswith("SET deleted = :deleted REMOVE "))- self.assertEqual(update["Update"]["ExpressionAttributeValues"], {":deleted": {"BOOL": True}})+ self.assertTrue(expression.startswith(prefix))+ self.assertEqual(update["Update"]["ExpressionAttributeValues"], values)self.assertEqual(sorted(expression.partition(" REMOVE ")[2].split(", ")), ["key_nonce", "sealed_key", "wrapped_key"])client.query.assert_not_called()- def test_team_deletion_closes_every_month_a_later_session_could_still_open(self) -> None:+ @parameterized.expand(+ [+ (+ "v3",+ V3KeyTarget,+ [+ (item_key("team:7", "image:2025-11"), None),+ (item_key("team:7", "image:2025-12"), None),+ (item_key("team:7", "image:2026-01"), None),+ ],+ ),+ (+ "v4",+ V4KeyTarget,+ [+ (item_key("team:7", "month:2025-11"), {"N": epoch("2026-04-01")}),+ (item_key("team:7", "month:2025-12"), {"N": epoch("2026-05-01")}),+ (item_key("team:7", "month:2026-01"), {"N": epoch("2026-06-01")}),+ ],+ ),+ ]+ )+ def test_team_deletion_closes_every_month_a_later_session_could_still_open(+ self, _name: str, target: TargetClass, expected: list[tuple[object, object]]+ ) -> None:client = MagicMock()- store = AITrainingPrivacyStore(client, "table")- with patch(- "products.ai_training.backend.privacy.store.timezone.now",- return_value=datetime.fromisoformat("2025-12-31T23:59:00+00:00"),- ):+ store = target(client, "table")+ with patch(NOW, return_value=datetime.fromisoformat("2025-12-31T23:59:00+00:00")):store.initialize(MagicMock(kind="team", team_id=7, identifiers=[]))updates = client.transact_write_items.call_args.kwargs["TransactItems"]self.assertEqual(- [update["Update"]["Key"] for update in updates],[- item_key("team:7", "image:2025-11"),- item_key("team:7", "image:2025-12"),- item_key("team:7", "image:2026-01"),+ (update["Update"]["Key"], update["Update"]["ExpressionAttributeValues"].get(":expiry"))+ for update in updates],+ expected,)- def test_team_deletion_completes_in_one_pass_and_shreds_every_shard_it_sweeps(self) -> None:+ @parameterized.expand([("v3", V3KeyTarget), ("v4", V4KeyTarget)])+ def test_team_deletion_completes_in_one_pass_and_shreds_every_shard_it_sweeps(+ self, _name: str, target: TargetClass+ ) -> None:request = MagicMock(kind="team", team_id=7, identifiers=[], cursor={}, completed_at=None)late_write = session_key(7, "01a09f92-e780-7000-8000-000000000003")client = MagicMock()client.query.side_effect = [{"Items": []}, {"Items": [late_write]}, *({"Items": []} for _ in range(31))]- store = AITrainingPrivacyStore(client, "table")+ store = AITrainingPrivacyStore([target(client, "table")])self.assertTrue(store.apply(request, time.monotonic() + 1))self.assertEqual(client.query.call_count, 33)shredded = [@@ -132,3 +181,102 @@ class TestAITrainingPrivacyStore(SimpleTestCase):self.assertIn(late_write, shredded)self.assertIsNotNone(request.completed_at)self.assertEqual(request.identifiers, [])++ def test_request_with_both_targets_yields_at_the_deadline_and_resumes_each_target(self) -> None:+ request = MagicMock(kind="team", team_id=7, identifiers=[], cursor={}, completed_at=None)+ clock = MagicMock(return_value=0.0)++ def v3_query(**kwargs: Any) -> DynamoResponse:+ if kwargs["ExpressionAttributeValues"][":pk"] == {"S": "team:7:shard:31"}:+ clock.return_value = 2.0+ return {"Items": []}++ v3_client = MagicMock()+ v3_client.query.side_effect = v3_query+ v4_client = MagicMock()+ old_month = item_key("team:7", "month:2026-08")++ def v4_query(**kwargs: Any) -> DynamoResponse:+ if kwargs["ExpressionAttributeValues"][":pk"] == {"S": "team:7"} and "ExclusiveStartKey" not in kwargs:+ clock.return_value = 2.0+ return {"Items": [old_month], "LastEvaluatedKey": old_month}+ return {"Items": []}++ v4_client.query.side_effect = v4_query+ store = AITrainingPrivacyStore([V3KeyTarget(v3_client, "v3"), V4KeyTarget(v4_client, "v4")])+ with (+ patch(NOW, return_value=datetime.fromisoformat("2026-10-21T12:00:00+00:00")),+ patch("products.ai_training.backend.privacy.store.time.monotonic", clock),+ ):+ self.assertFalse(store.apply(request, 1.0))+ self.assertEqual(request.cursor["work"], [])+ self.assertNotIn("v4_work", request.cursor)+ v4_client.transact_write_items.assert_not_called()+ clock.return_value = 0.0+ self.assertFalse(store.apply(request, 1.0))+ self.assertIsNone(request.completed_at)+ self.assertEqual(request.cursor["v4_work"], [{"op": "team", "team_id": 7, "shard": -1, "after": old_month}])+ clock.return_value = 0.0+ self.assertTrue(store.apply(request, 1.0))+ self.assertIsNotNone(request.completed_at)+ self.assertEqual(v3_client.query.call_count, 33)+ self.assertEqual(v4_client.query.call_count, 34)+ self.assertEqual(v4_client.query.call_args_list[1].kwargs["ExclusiveStartKey"], old_month)+ tombstones = [+ (update["Update"]["Key"], update["Update"]["ExpressionAttributeValues"][":expiry"])+ for call in v4_client.transact_write_items.call_args_list+ for update in call.kwargs["TransactItems"]+ ]+ self.assertIn((old_month, {"N": epoch("2027-01-01")}), tombstones)++ def test_request_in_flight_with_only_v3_progress_resumes_v3_and_runs_v4_from_the_start(self) -> None:+ request = MagicMock(+ kind="team",+ team_id=7,+ identifiers=[],+ cursor={"work": [{"op": "team", "team_id": 7, "shard": 31}]},+ completed_at=None,+ )+ v3_client = MagicMock()+ v3_client.query.return_value = {"Items": []}+ v4_client = MagicMock()+ v4_client.query.return_value = {"Items": []}+ store = AITrainingPrivacyStore([V3KeyTarget(v3_client, "v3"), V4KeyTarget(v4_client, "v4")])+ with patch(NOW, return_value=datetime.fromisoformat("2026-10-21T12:00:00+00:00")):+ self.assertTrue(store.apply(request, time.monotonic() + 1))+ self.assertEqual(v3_client.query.call_count, 1)+ v3_client.transact_write_items.assert_not_called()+ self.assertEqual(v4_client.query.call_count, 33)+ self.assertEqual(+ [update["Update"]["Key"] for update in v4_client.transact_write_items.call_args.kwargs["TransactItems"]],+ [item_key("team:7", f"month:{month}") for month in ("2026-09", "2026-10", "2026-11")],+ )+ self.assertIsNotNone(request.completed_at)++ @parameterized.expand(+ [+ ("v3", "v3-table", "", [("v3-table", "us-east-1")]),+ ("v4", "", "v4-table", [("v4-table", "eu-west-1")]),+ ("both", "v3-table", "v4-table", [("v3-table", "us-east-1"), ("v4-table", "eu-west-1")]),+ ]+ )+ def test_store_runs_one_target_per_configured_table_with_its_own_client(+ self, _name: str, v3_table: str, v4_table: str, expected: list[tuple[str, str]]+ ) -> None:+ with (+ self.settings(+ AI_RESEARCH_REPLAY_KEY_TABLE=v3_table,+ AI_RESEARCH_REPLAY_AWS_REGION="us-east-1",+ AI_RESEARCH_V4_KEY_TABLE=v4_table,+ AI_RESEARCH_V4_AWS_REGION="eu-west-1",+ ),+ patch("products.ai_training.backend.privacy.keys.boto3.client") as client,+ ):+ store = AITrainingPrivacyStore.from_settings()+ self.assertEqual(+ [+ (target.table_name, call.kwargs["region_name"])+ for target, call in zip(store.targets, client.call_args_list, strict=True)+ ],+ expected,+ )