Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(ai-training): shred v4 replay keys on deletion (#121247)

Nicholas Waltz37565421b3e6
diff --git a/posthog/settings/ingestion.py b/posthog/settings/ingestion.py
index 22d873dce765..3984a01c444d 100644
--- a/posthog/settings/ingestion.py
+++ b/posthog/settings/ingestion.py
@@ -160,3 +160,6 @@ AI_RESEARCH_REPLAY_KEY_TABLE = os.getenv("AI_RESEARCH_REPLAY_KEY_TABLE", "")
AI_RESEARCH_REPLAY_AWS_REGION = os.getenv("AI_RESEARCH_REPLAY_AWS_REGION", "us-east-1")
AI_RESEARCH_REPLAY_DYNAMODB_ENDPOINT = os.getenv("AI_RESEARCH_REPLAY_DYNAMODB_ENDPOINT", "")
AI_RESEARCH_REPLAY_KMS_KEY_ARN = os.getenv("AI_RESEARCH_REPLAY_KMS_KEY_ARN", "")
+AI_RESEARCH_V4_KEY_TABLE = os.getenv("AI_RESEARCH_V4_KEY_TABLE", "")
+AI_RESEARCH_V4_AWS_REGION = os.getenv("AI_RESEARCH_V4_AWS_REGION", "us-east-1")
+AI_RESEARCH_V4_DYNAMODB_ENDPOINT = os.getenv("AI_RESEARCH_V4_DYNAMODB_ENDPOINT", "")
diff --git a/products/ai_training/backend/config.py b/products/ai_training/backend/config.py
index 9a4918b38a13..a113c78377c0 100644
--- a/products/ai_training/backend/config.py
+++ b/products/ai_training/backend/config.py
@@ -5,5 +5,9 @@ def key_table_name() -> str:
return str(getattr(settings, "AI_RESEARCH_REPLAY_KEY_TABLE", ""))
+def v4_key_table_name() -> str:
+ return str(getattr(settings, "AI_RESEARCH_V4_KEY_TABLE", ""))
+
+
def privacy_enabled() -> bool:
- return bool(key_table_name())
+ return bool(key_table_name() or v4_key_table_name())
diff --git a/products/ai_training/backend/management/commands/delete_ai_training_month.py b/products/ai_training/backend/management/commands/delete_ai_training_month.py
index baa59cf5e02e..eef1c6033f17 100644
--- a/products/ai_training/backend/management/commands/delete_ai_training_month.py
+++ b/products/ai_training/backend/management/commands/delete_ai_training_month.py
@@ -4,7 +4,7 @@ from typing import Any
from django.core.management.base import BaseCommand, CommandError
from products.ai_training.backend.config import key_table_name
-from products.ai_training.backend.privacy.store import AITrainingPrivacyStore
+from products.ai_training.backend.privacy.store import V3KeyTarget
class Command(BaseCommand):
@@ -17,7 +17,7 @@ class Command(BaseCommand):
if not key_table_name():
raise CommandError("AI_RESEARCH_REPLAY_KEY_TABLE is not configured")
try:
- count = AITrainingPrivacyStore.from_settings().delete_month(options["session_month"])
+ count = V3KeyTarget.from_settings().delete_month(options["session_month"])
except ValueError as error:
raise CommandError(str(error)) from error
self.stdout.write(f"Removed {count} indexed keys. A reader that already cached a key can still use it.")
diff --git a/products/ai_training/backend/privacy/keys.py b/products/ai_training/backend/privacy/keys.py
new file mode 100644
index 000000000000..460efbaeb16c
--- /dev/null
+++ b/products/ai_training/backend/privacy/keys.py
@@ -0,0 +1,92 @@
+import hashlib
+from datetime import UTC, datetime, timedelta
+from typing import Protocol, TypedDict, cast
+
+import boto3
+from botocore.config import Config
+
+from posthog.dataclasses import frozen
+
+from products.ai_training.backend.models import AITrainingDeletionRequest
+
+KEY_SHARDS = 32
+# A per-session deletion keeps the team month key, so it must also remove the seal that the month key opens.
+KEY_MATERIAL_ATTRIBUTES = ("wrapped_key", "sealed_key", "key_nonce")
+DynamoItem = dict[str, dict[str, str | bool | bytes]]
+
+
+class DynamoResponse(TypedDict, total=False):
+ Item: DynamoItem
+ Items: list[DynamoItem]
+ LastEvaluatedKey: DynamoItem
+
+
+class DeletionWork(TypedDict, total=False):
+ op: str
+ team_id: int
+ shard: int
+ after: DynamoItem
+
+
+class PrivacyDynamoClient(Protocol):
+ def put_item(self, **kwargs: object) -> DynamoResponse: ...
+ def get_item(self, **kwargs: object) -> DynamoResponse: ...
+ def query(self, **kwargs: object) -> DynamoResponse: ...
+ def transact_write_items(self, **kwargs: object) -> DynamoResponse: ...
+
+
+class DeletionTarget(Protocol):
+ # The request cursor stores each target's work under this key, so targets progress independently.
+ cursor_key: str
+ table_name: str
+
+ def initialize(self, request: AITrainingDeletionRequest) -> list[DeletionWork]: ...
+ def advance(self, work: DeletionWork) -> list[DeletionWork]: ...
+
+
+def dynamo_client(region: str, endpoint: str) -> PrivacyDynamoClient:
+ client = boto3.client(
+ "dynamodb",
+ region_name=region,
+ endpoint_url=endpoint or None,
+ config=Config(connect_timeout=5, read_timeout=5, retries={"max_attempts": 4, "mode": "standard"}),
+ )
+ return cast(PrivacyDynamoClient, client)
+
+
+def item_key(pk: str, sk: str) -> DynamoItem:
+ return {"pk": {"S": pk}, "sk": {"S": sk}}
+
+
+def month_end(session_month: str) -> datetime:
+ year, month = (int(part) for part in session_month.split("-"))
+ return datetime(year + month // 12, month % 12 + 1, 1, tzinfo=UTC)
+
+
+def identity_digest(value: str) -> str:
+ return hashlib.sha256(value.encode()).hexdigest()
+
+
+def session_key(team_id: int, session_id: str) -> DynamoItem:
+ shard = int(identity_digest(session_id)[:8], 16) % KEY_SHARDS
+ return item_key(f"team:{team_id}:shard:{shard}", f"session:{session_id}")
+
+
+@frozen
+class ReachableMonths:
+ previous: str
+ current: str
+ following: str
+
+ @property
+ def months(self) -> list[str]:
+ return [self.previous, self.current, self.following]
+
+
+def reachable_months(now: datetime) -> ReachableMonths:
+ current = f"{now:%Y-%m}"
+ return ReachableMonths(
+ previous=f"{datetime(now.year, now.month, 1, tzinfo=UTC) - timedelta(days=1):%Y-%m}",
+ current=current,
+ following=f"{month_end(current):%Y-%m}",
+ )
diff --git a/products/ai_training/backend/privacy/store.py b/products/ai_training/backend/privacy/store.py
index 3766b030f439..85be122b2710 100644
--- a/products/ai_training/backend/privacy/store.py
+++ b/products/ai_training/backend/privacy/store.py
@@ -1,85 +1,51 @@
import re
import time
-import hashlib
from collections.abc import Sequence
-from datetime import UTC, datetime, timedelta
-from typing import Protocol, TypedDict, cast
+from datetime import timedelta
from django.conf import settings
from django.db import transaction
from django.utils import timezone
-import boto3
import structlog
-from botocore.config import Config
-from products.ai_training.backend.config import key_table_name
+from products.ai_training.backend.config import key_table_name, v4_key_table_name
from products.ai_training.backend.models import AITrainingDeletionRequest
+from products.ai_training.backend.privacy.keys import (
+ KEY_MATERIAL_ATTRIBUTES,
+ KEY_SHARDS,
+ DeletionTarget,
+ DeletionWork,
+ DynamoItem,
+ DynamoResponse,
+ PrivacyDynamoClient,
+ dynamo_client,
+ item_key,
+ month_end,
+ reachable_months,
+ session_key,
+)
+from products.ai_training.backend.privacy.v4 import V4KeyTarget
logger = structlog.get_logger(__name__)
-KEY_SHARDS = 32
# Equals ML_SESSION_MAX_AGE_DAYS in nodejs/src/ingestion/pipelines/sessionreplay/ml-mirror/session-identifier-format.ts: ingestion drops sessions that started earlier than that, so no key for a month can appear after the month end plus this period.
MONTH_DELETE_GRACE_DAYS = 14
# A batch admitted just inside the grace period still commits within its 45 s budget, so deletion stays behind that too.
MONTH_DELETE_IN_FLIGHT_MARGIN = timedelta(hours=1)
-# A per-session deletion keeps the team month key, so it must also remove the seal that the month key opens.
-KEY_MATERIAL_ATTRIBUTES = ("wrapped_key", "sealed_key", "key_nonce")
-DynamoItem = dict[str, dict[str, str | bool | bytes]]
-class DynamoResponse(TypedDict, total=False):
- Item: DynamoItem
- Items: list[DynamoItem]
- LastEvaluatedKey: DynamoItem
+class V3KeyTarget:
+ cursor_key = "work"
-
-class DeletionWork(TypedDict, total=False):
- op: str
- team_id: int
- shard: int
- after: DynamoItem
-
-
-class PrivacyDynamoClient(Protocol):
- def put_item(self, **kwargs: object) -> DynamoResponse: ...
- def get_item(self, **kwargs: object) -> DynamoResponse: ...
- def query(self, **kwargs: object) -> DynamoResponse: ...
- def transact_write_items(self, **kwargs: object) -> DynamoResponse: ...
-
-
-def item_key(pk: str, sk: str) -> DynamoItem:
- return {"pk": {"S": pk}, "sk": {"S": sk}}
-
-
-def month_end(session_month: str) -> datetime:
- year, month = (int(part) for part in session_month.split("-"))
- return datetime(year + month // 12, month % 12 + 1, 1, tzinfo=UTC)
-
-
-def identity_digest(value: str) -> str:
- return hashlib.sha256(value.encode()).hexdigest()
-
-
-def session_key(team_id: int, session_id: str) -> DynamoItem:
- shard = int(identity_digest(session_id)[:8], 16) % KEY_SHARDS
- return item_key(f"team:{team_id}:shard:{shard}", f"session:{session_id}")
-
-
-class AITrainingPrivacyStore:
def __init__(self, client: PrivacyDynamoClient, table_name: str) -> None:
self.client = client
self.table_name = table_name
@classmethod
- def from_settings(cls) -> "AITrainingPrivacyStore":
- client = boto3.client(
- "dynamodb",
- region_name=settings.AI_RESEARCH_REPLAY_AWS_REGION,
- endpoint_url=settings.AI_RESEARCH_REPLAY_DYNAMODB_ENDPOINT or None,
- config=Config(connect_timeout=5, read_timeout=5, retries={"max_attempts": 4, "mode": "standard"}),
- )
- return cls(cast(PrivacyDynamoClient, client), key_table_name())
+ def from_settings(cls) -> "V3KeyTarget":
+ client = dynamo_client(settings.AI_RESEARCH_REPLAY_AWS_REGION, settings.AI_RESEARCH_REPLAY_DYNAMODB_ENDPOINT)
+ return cls(client, key_table_name())
def delete_month(self, session_month: str) -> int:
if re.fullmatch(r"[0-9]{4}-(0[1-9]|1[0-2])", session_month) is None:
@@ -138,11 +104,7 @@ class AITrainingPrivacyStore:
# A sweep reaches only an image key that already exists, so a team that keeps sending would open a month the sweep never saw. Ingestion admits a session up to ML_SESSION_MAX_AGE_DAYS old, which is inside one month, so these three are every month a later session can still open. An update creates the row, so each one closes whether or not it holds a key.
def shred_reachable_months(self, team_id: int) -> None:
- now = timezone.now()
- current = f"{now:%Y-%m}"
- previous = f"{datetime(now.year, now.month, 1, tzinfo=UTC) - timedelta(days=1):%Y-%m}"
- months = (previous, current, f"{month_end(current):%Y-%m}")
- self.shred([item_key(f"team:{team_id}", f"image:{month}") for month in months])
+ self.shred([item_key(f"team:{team_id}", f"image:{month}") for month in reachable_months(timezone.now()).months])
def initialize(self, request: AITrainingDeletionRequest) -> list[DeletionWork]:
if request.team_id is None:
@@ -173,16 +135,34 @@ class AITrainingPrivacyStore:
return [next_work]
return []
+
+class AITrainingPrivacyStore:
+ def __init__(self, targets: Sequence[DeletionTarget]) -> None:
+ self.targets = list(targets)
+
+ @classmethod
+ def from_settings(cls) -> "AITrainingPrivacyStore":
+ targets: list[DeletionTarget] = []
+ if key_table_name():
+ targets.append(V3KeyTarget.from_settings())
+ if v4_key_table_name():
+ targets.append(V4KeyTarget.from_settings())
+ return cls(targets)
+
def apply(self, request: AITrainingDeletionRequest, deadline: float) -> bool:
- work = request.cursor.get("work")
- if work is None:
- work = self.initialize(request)
- self.save_cursor(request, work=work)
- while work:
- if time.monotonic() >= deadline:
- return False
- work = self.advance(work[0]) + work[1:]
- self.save_cursor(request, work=work)
+ for target in self.targets:
+ work = request.cursor.get(target.cursor_key)
+ if work is None:
+ # Initialization writes with retries, so a target that starts after the deadline can overrun the task time limit.
+ if time.monotonic() >= deadline:
+ return False
+ work = target.initialize(request)
+ self.save_cursor(request, **{target.cursor_key: work})
+ while work:
+ if time.monotonic() >= deadline:
+ return False
+ work = target.advance(work[0]) + work[1:]
+ self.save_cursor(request, **{target.cursor_key: work})
# A key that a batch stores after the sweep passes its shard is sealed under a month key this deletion already tombstoned, so a second sweep finds nothing that is readable. The sweep removes rows to save cost. See products/ai_training/docs/replay-data.md.
request.completed_at = timezone.now()
request.identifiers = []
diff --git a/products/ai_training/backend/privacy/v4.py b/products/ai_training/backend/privacy/v4.py
new file mode 100644
index 000000000000..c0f7fa820f5c
--- /dev/null
+++ b/products/ai_training/backend/privacy/v4.py
@@ -0,0 +1,120 @@
+from collections.abc import Sequence
+from datetime import UTC, datetime
+
+from django.conf import settings
+from django.utils import timezone
+
+from products.ai_training.backend.config import v4_key_table_name
+from products.ai_training.backend.models import AITrainingDeletionRequest
+from products.ai_training.backend.privacy.keys import (
+ KEY_MATERIAL_ATTRIBUTES,
+ KEY_SHARDS,
+ DeletionWork,
+ DynamoItem,
+ DynamoResponse,
+ PrivacyDynamoClient,
+ dynamo_client,
+ item_key,
+ reachable_months,
+ session_key,
+)
+
+# Equals the expiry rule of the v4 recorder: a row lives until the start of the fifth month after its key month.
+KEY_RETENTION_MONTHS = 5
+
+
+def key_expiry(key_month: str) -> int:
+ year, month = (int(part) for part in key_month.split("-"))
+ index = year * 12 + month - 1 + KEY_RETENTION_MONTHS
+ return int(datetime(index // 12, index % 12 + 1, 1, tzinfo=UTC).timestamp())
+
+
+class V4KeyTarget:
+ cursor_key = "v4_work"
+
+ def __init__(self, client: PrivacyDynamoClient, table_name: str) -> None:
+ self.client = client
+ self.table_name = table_name
+
+ @classmethod
+ def from_settings(cls) -> "V4KeyTarget":
+ client = dynamo_client(settings.AI_RESEARCH_V4_AWS_REGION, settings.AI_RESEARCH_V4_DYNAMODB_ENDPOINT)
+ return cls(client, v4_key_table_name())
+
+ def page(self, pk: str, prefix: str, cursor: DynamoItem | None = None) -> DynamoResponse:
+ return self.client.query(
+ TableName=self.table_name,
+ KeyConditionExpression="pk = :pk AND begins_with(sk, :prefix)",
+ ExpressionAttributeValues={":pk": {"S": pk}, ":prefix": {"S": prefix}},
+ ProjectionExpression="pk, sk",
+ ConsistentRead=True,
+ Limit=50,
+ **({"ExclusiveStartKey": cursor} if cursor else {}),
+ )
+
+ # An update creates a row that does not exist yet, and writers never overwrite a row, so the tombstone needs an expiry or it stays forever. if_not_exists keeps the expiry of a row the recorder wrote.
+ def tombstone(self, rows: Sequence[tuple[DynamoItem, int]]) -> None:
+ unique = {str(key["pk"]["S"]) + "\0" + str(key["sk"]["S"]): (key, expiry) for key, expiry in rows}
+ values = list(unique.values())
+ for offset in range(0, len(values), 100):
+ self.client.transact_write_items(
+ TransactItems=[
+ {
+ "Update": {
+ "TableName": self.table_name,
+ "Key": {"pk": key["pk"], "sk": key["sk"]},
+ "UpdateExpression": "SET deleted = :true, expires_at = if_not_exists(expires_at, :expiry) REMOVE "
+ + ", ".join(KEY_MATERIAL_ATTRIBUTES),
+ "ExpressionAttributeValues": {":true": {"BOOL": True}, ":expiry": {"N": str(expiry)}},
+ },
+ }
+ for key, expiry in values[offset : offset + 100]
+ ]
+ )
+
+ def month_row(self, team_id: int, key_month: str) -> tuple[DynamoItem, int]:
+ return item_key(f"team:{team_id}", f"month:{key_month}"), key_expiry(key_month)
+
+ def session_expiry(self) -> int:
+ return key_expiry(f"{timezone.now():%Y-%m}")
+
+ def initialize(self, request: AITrainingDeletionRequest) -> list[DeletionWork]:
+ if request.team_id is None:
+ raise ValueError("AI training deletion request has no team")
+ team_id = request.team_id
+ if request.kind == "team":
+ # A session can still open these months after the sweep passes, and a tombstone stops the recorder from writing a key there.
+ self.tombstone([self.month_row(team_id, month) for month in reachable_months(timezone.now()).months])
+ return [{"op": "team", "team_id": team_id, "shard": -1}]
+ if request.kind == "session":
+ expiry = self.session_expiry()
+ self.tombstone([(session_key(team_id, value), expiry) for value in request.identifiers])
+ return []
+ raise ValueError("Unknown AI training deletion request")
+
+ def advance(self, work: DeletionWork) -> list[DeletionWork]:
+ if work["op"] != "team":
+ raise ValueError("Unknown AI training deletion operation")
+ team_id = int(work["team_id"])
+ shard = int(work["shard"])
+ if shard == -1:
+ response = self.page(f"team:{team_id}", "month:", work.get("after"))
+ self.tombstone(
+ [
+ self.month_row(team_id, str(row["sk"]["S"]).removeprefix("month:"))
+ for row in response.get("Items", [])
+ ]
+ )
+ else:
+ response = self.page(f"team:{team_id}:shard:{shard}", "session:", work.get("after"))
+ expiry = self.session_expiry()
+ self.tombstone(
+ [(item_key(str(row["pk"]["S"]), str(row["sk"]["S"])), expiry) for row in response.get("Items", [])]
+ )
+ if response.get("LastEvaluatedKey"):
+ return [{**work, "after": response["LastEvaluatedKey"]}]
+ if shard + 1 < KEY_SHARDS:
+ next_work: DeletionWork = {**work, "shard": shard + 1}
+ next_work.pop("after", None)
+ return [next_work]
+ return []
diff --git a/products/ai_training/backend/tests/test_privacy_store.py b/products/ai_training/backend/tests/test_privacy_store.py
index 3adcb96f316b..75216d48dd45 100644
--- a/products/ai_training/backend/tests/test_privacy_store.py
+++ b/products/ai_training/backend/tests/test_privacy_store.py
@@ -1,5 +1,6 @@
import time
from datetime import datetime
+from typing import Any
from unittest.mock import MagicMock, patch
@@ -7,17 +8,28 @@ from django.test import SimpleTestCase
from parameterized import parameterized
-from products.ai_training.backend.privacy.store import AITrainingPrivacyStore, DynamoResponse, item_key, session_key
+from products.ai_training.backend.privacy.keys import DynamoResponse, item_key, session_key
+from products.ai_training.backend.privacy.store import AITrainingPrivacyStore, V3KeyTarget
+from products.ai_training.backend.privacy.v4 import V4KeyTarget
from products.ai_training.backend.tasks.tasks import process_ai_training_privacy_requests
+NOW = "products.ai_training.backend.privacy.store.timezone.now"
+TargetClass = type[V3KeyTarget] | type[V4KeyTarget]
+
+
+def epoch(day: str) -> str:
+ return str(int(datetime.fromisoformat(f"{day}T00:00:00+00:00").timestamp()))
+
class TestAITrainingPrivacyStore(SimpleTestCase):
- @parameterized.expand([("disabled", "", False), ("enabled", "test-table", True)])
+ @parameterized.expand(
+ [("disabled", "", "", False), ("v3", "test-table", "", True), ("v4", "", "test-v4-table", True)]
+ )
def test_privacy_task_uses_dedicated_queue_and_only_drains_when_enabled(
- self, _name: str, table: str, enabled: bool
+ self, _name: str, table: str, v4_table: str, enabled: bool
) -> None:
with (
- self.settings(AI_RESEARCH_REPLAY_KEY_TABLE=table),
+ self.settings(AI_RESEARCH_REPLAY_KEY_TABLE=table, AI_RESEARCH_V4_KEY_TABLE=v4_table),
patch("products.ai_training.backend.tasks.tasks.AITrainingPrivacyStore.from_settings") as store,
):
signature = process_ai_training_privacy_requests.signature()
@@ -46,7 +58,7 @@ class TestAITrainingPrivacyStore(SimpleTestCase):
return pages.pop(0)
client.query.side_effect = query
- store = AITrainingPrivacyStore(client, "table")
+ store = V3KeyTarget(client, "table")
self.assertEqual(store.delete_month("2025-09"), 2)
client.put_item.assert_not_called()
self.assertEqual(
@@ -74,54 +86,91 @@ class TestAITrainingPrivacyStore(SimpleTestCase):
) -> None:
client = MagicMock()
client.query.return_value = {"Items": []}
- store = AITrainingPrivacyStore(client, "table")
- with patch("products.ai_training.backend.privacy.store.timezone.now", return_value=datetime.fromisoformat(now)):
+ store = V3KeyTarget(client, "table")
+ with patch(NOW, return_value=datetime.fromisoformat(now)):
if allowed:
self.assertEqual(store.delete_month(month), 0)
else:
with self.assertRaisesRegex(ValueError, "can be deleted from"):
store.delete_month(month)
- def test_session_deletion_shreds_every_key_attribute_without_querying_user_indexes(self) -> None:
+ @parameterized.expand(
+ [
+ ("v3", V3KeyTarget, "SET deleted = :deleted REMOVE ", {":deleted": {"BOOL": True}}),
+ (
+ "v4",
+ V4KeyTarget,
+ "SET deleted = :true, expires_at = if_not_exists(expires_at, :expiry) REMOVE ",
+ {":true": {"BOOL": True}, ":expiry": {"N": epoch("2027-03-01")}},
+ ),
+ ]
+ )
+ def test_session_deletion_shreds_every_key_attribute_without_querying_user_indexes(
+ self, _name: str, target: TargetClass, prefix: str, values: dict[str, dict[str, object]]
+ ) -> None:
client = MagicMock()
sessions = ["01a09f92-e780-7000-8000-000000000001", "01a09f92-e780-7000-8000-000000000002"]
- store = AITrainingPrivacyStore(client, "table")
- self.assertEqual(store.initialize(MagicMock(kind="session", team_id=7, identifiers=sessions)), [])
+ store = target(client, "table")
+ with patch(NOW, return_value=datetime.fromisoformat("2026-10-31T23:59:00+00:00")):
+ self.assertEqual(store.initialize(MagicMock(kind="session", team_id=7, identifiers=sessions)), [])
updates = client.transact_write_items.call_args.kwargs["TransactItems"]
self.assertEqual([update["Update"]["Key"] for update in updates], [session_key(7, value) for value in sessions])
for update in updates:
expression = update["Update"]["UpdateExpression"]
- self.assertTrue(expression.startswith("SET deleted = :deleted REMOVE "))
- self.assertEqual(update["Update"]["ExpressionAttributeValues"], {":deleted": {"BOOL": True}})
+ self.assertTrue(expression.startswith(prefix))
+ self.assertEqual(update["Update"]["ExpressionAttributeValues"], values)
self.assertEqual(
sorted(expression.partition(" REMOVE ")[2].split(", ")), ["key_nonce", "sealed_key", "wrapped_key"]
)
client.query.assert_not_called()
- def test_team_deletion_closes_every_month_a_later_session_could_still_open(self) -> None:
+ @parameterized.expand(
+ [
+ (
+ "v3",
+ V3KeyTarget,
+ [
+ (item_key("team:7", "image:2025-11"), None),
+ (item_key("team:7", "image:2025-12"), None),
+ (item_key("team:7", "image:2026-01"), None),
+ ],
+ ),
+ (
+ "v4",
+ V4KeyTarget,
+ [
+ (item_key("team:7", "month:2025-11"), {"N": epoch("2026-04-01")}),
+ (item_key("team:7", "month:2025-12"), {"N": epoch("2026-05-01")}),
+ (item_key("team:7", "month:2026-01"), {"N": epoch("2026-06-01")}),
+ ],
+ ),
+ ]
+ )
+ def test_team_deletion_closes_every_month_a_later_session_could_still_open(
+ self, _name: str, target: TargetClass, expected: list[tuple[object, object]]
+ ) -> None:
client = MagicMock()
- store = AITrainingPrivacyStore(client, "table")
- with patch(
- "products.ai_training.backend.privacy.store.timezone.now",
- return_value=datetime.fromisoformat("2025-12-31T23:59:00+00:00"),
- ):
+ store = target(client, "table")
+ with patch(NOW, return_value=datetime.fromisoformat("2025-12-31T23:59:00+00:00")):
store.initialize(MagicMock(kind="team", team_id=7, identifiers=[]))
updates = client.transact_write_items.call_args.kwargs["TransactItems"]
self.assertEqual(
- [update["Update"]["Key"] for update in updates],
[
- item_key("team:7", "image:2025-11"),
- item_key("team:7", "image:2025-12"),
- item_key("team:7", "image:2026-01"),
+ (update["Update"]["Key"], update["Update"]["ExpressionAttributeValues"].get(":expiry"))
+ for update in updates
],
+ expected,
)
- def test_team_deletion_completes_in_one_pass_and_shreds_every_shard_it_sweeps(self) -> None:
+ @parameterized.expand([("v3", V3KeyTarget), ("v4", V4KeyTarget)])
+ def test_team_deletion_completes_in_one_pass_and_shreds_every_shard_it_sweeps(
+ self, _name: str, target: TargetClass
+ ) -> None:
request = MagicMock(kind="team", team_id=7, identifiers=[], cursor={}, completed_at=None)
late_write = session_key(7, "01a09f92-e780-7000-8000-000000000003")
client = MagicMock()
client.query.side_effect = [{"Items": []}, {"Items": [late_write]}, *({"Items": []} for _ in range(31))]
- store = AITrainingPrivacyStore(client, "table")
+ store = AITrainingPrivacyStore([target(client, "table")])
self.assertTrue(store.apply(request, time.monotonic() + 1))
self.assertEqual(client.query.call_count, 33)
shredded = [
@@ -132,3 +181,102 @@ class TestAITrainingPrivacyStore(SimpleTestCase):
self.assertIn(late_write, shredded)
self.assertIsNotNone(request.completed_at)
self.assertEqual(request.identifiers, [])
+
+ def test_request_with_both_targets_yields_at_the_deadline_and_resumes_each_target(self) -> None:
+ request = MagicMock(kind="team", team_id=7, identifiers=[], cursor={}, completed_at=None)
+ clock = MagicMock(return_value=0.0)
+
+ def v3_query(**kwargs: Any) -> DynamoResponse:
+ if kwargs["ExpressionAttributeValues"][":pk"] == {"S": "team:7:shard:31"}:
+ clock.return_value = 2.0
+ return {"Items": []}
+
+ v3_client = MagicMock()
+ v3_client.query.side_effect = v3_query
+ v4_client = MagicMock()
+ old_month = item_key("team:7", "month:2026-08")
+
+ def v4_query(**kwargs: Any) -> DynamoResponse:
+ if kwargs["ExpressionAttributeValues"][":pk"] == {"S": "team:7"} and "ExclusiveStartKey" not in kwargs:
+ clock.return_value = 2.0
+ return {"Items": [old_month], "LastEvaluatedKey": old_month}
+ return {"Items": []}
+
+ v4_client.query.side_effect = v4_query
+ store = AITrainingPrivacyStore([V3KeyTarget(v3_client, "v3"), V4KeyTarget(v4_client, "v4")])
+ with (
+ patch(NOW, return_value=datetime.fromisoformat("2026-10-21T12:00:00+00:00")),
+ patch("products.ai_training.backend.privacy.store.time.monotonic", clock),
+ ):
+ self.assertFalse(store.apply(request, 1.0))
+ self.assertEqual(request.cursor["work"], [])
+ self.assertNotIn("v4_work", request.cursor)
+ v4_client.transact_write_items.assert_not_called()
+ clock.return_value = 0.0
+ self.assertFalse(store.apply(request, 1.0))
+ self.assertIsNone(request.completed_at)
+ self.assertEqual(request.cursor["v4_work"], [{"op": "team", "team_id": 7, "shard": -1, "after": old_month}])
+ clock.return_value = 0.0
+ self.assertTrue(store.apply(request, 1.0))
+ self.assertIsNotNone(request.completed_at)
+ self.assertEqual(v3_client.query.call_count, 33)
+ self.assertEqual(v4_client.query.call_count, 34)
+ self.assertEqual(v4_client.query.call_args_list[1].kwargs["ExclusiveStartKey"], old_month)
+ tombstones = [
+ (update["Update"]["Key"], update["Update"]["ExpressionAttributeValues"][":expiry"])
+ for call in v4_client.transact_write_items.call_args_list
+ for update in call.kwargs["TransactItems"]
+ ]
+ self.assertIn((old_month, {"N": epoch("2027-01-01")}), tombstones)
+
+ def test_request_in_flight_with_only_v3_progress_resumes_v3_and_runs_v4_from_the_start(self) -> None:
+ request = MagicMock(
+ kind="team",
+ team_id=7,
+ identifiers=[],
+ cursor={"work": [{"op": "team", "team_id": 7, "shard": 31}]},
+ completed_at=None,
+ )
+ v3_client = MagicMock()
+ v3_client.query.return_value = {"Items": []}
+ v4_client = MagicMock()
+ v4_client.query.return_value = {"Items": []}
+ store = AITrainingPrivacyStore([V3KeyTarget(v3_client, "v3"), V4KeyTarget(v4_client, "v4")])
+ with patch(NOW, return_value=datetime.fromisoformat("2026-10-21T12:00:00+00:00")):
+ self.assertTrue(store.apply(request, time.monotonic() + 1))
+ self.assertEqual(v3_client.query.call_count, 1)
+ v3_client.transact_write_items.assert_not_called()
+ self.assertEqual(v4_client.query.call_count, 33)
+ self.assertEqual(
+ [update["Update"]["Key"] for update in v4_client.transact_write_items.call_args.kwargs["TransactItems"]],
+ [item_key("team:7", f"month:{month}") for month in ("2026-09", "2026-10", "2026-11")],
+ )
+ self.assertIsNotNone(request.completed_at)
+
+ @parameterized.expand(
+ [
+ ("v3", "v3-table", "", [("v3-table", "us-east-1")]),
+ ("v4", "", "v4-table", [("v4-table", "eu-west-1")]),
+ ("both", "v3-table", "v4-table", [("v3-table", "us-east-1"), ("v4-table", "eu-west-1")]),
+ ]
+ )
+ def test_store_runs_one_target_per_configured_table_with_its_own_client(
+ self, _name: str, v3_table: str, v4_table: str, expected: list[tuple[str, str]]
+ ) -> None:
+ with (
+ self.settings(
+ AI_RESEARCH_REPLAY_KEY_TABLE=v3_table,
+ AI_RESEARCH_REPLAY_AWS_REGION="us-east-1",
+ AI_RESEARCH_V4_KEY_TABLE=v4_table,
+ AI_RESEARCH_V4_AWS_REGION="eu-west-1",
+ ),
+ patch("products.ai_training.backend.privacy.keys.boto3.client") as client,
+ ):
+ store = AITrainingPrivacyStore.from_settings()
+ self.assertEqual(
+ [
+ (target.table_name, call.kwargs["region_name"])
+ for target, call in zip(store.targets, client.call_args_list, strict=True)
+ ],
+ expected,
+ )