Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Address review on data retention
Petr Bauchdf1c243d6878
diff --git a/core/switch_core/db/models.py b/core/switch_core/db/models.pyindex 3fc22a574..c16e98930 100644--- a/core/switch_core/db/models.py+++ b/core/switch_core/db/models.py@@ -2462,6 +2462,12 @@ class BridgeMessageMap(TenantScoped, Base):__tablename__ = "bridge_message_map"__table_args__ = (Index("ix_bridge_message_map_tenant_id", "tenant_id"),+ # Retention forgets the mappings of the messages it deletes, by event id.+ Index(+ "ix_bridge_message_map_tenant_event",+ "tenant_id",+ "transport_event_id",+ ),UniqueConstraint("bridge_id", "transport_event_id"),UniqueConstraint("bridge_id", "external_post_id"),ForeignKeyConstraint(@@ -2708,11 +2714,11 @@ class TrustSettings(Base):class Message(TenantScoped, Base):"""A message as it was sent into a room.- Written alongside the send to the message bus, which remains the source of- truth for history until the read path moves here. Rows are therefore a- parallel record, not yet an authoritative one: a write that fails after a- successful send leaves a gap, by design, so that a database problem cannot- make messaging less reliable.+ This table is the message bus: a send is an insert here, and `seq` is the+ position every delivery cursor reads from. It is also the room's history,+ and the only copy Switch keeps, so deleting a row (data retention, erasing+ a person) deletes the message for every reader. `rooms.seq_floor` keeps+ numbering intact when the deleted rows were a room's newest.Every participant in a room is a Switch-owned client, so recording eachsend captures the whole room exactly once — including messages a human@@ -2823,6 +2829,9 @@ class MessageAttachment(TenantScoped, Base):__tablename__ = "message_attachments"__table_args__ = (Index("ix_message_attachments_tenant_id", "tenant_id"),+ # The file sweep asks, for every stored file, whether any attachment+ # still names it.+ Index("ix_message_attachments_tenant_uri", "tenant_id", "uri"),Index("ix_message_attachments_message", "message_id"),ForeignKeyConstraint(["tenant_id", "message_id"],diff --git a/core/switch_core/db/stores/retention_store.py b/core/switch_core/db/stores/retention_store.pyindex 3909aae52..3573b76a6 100644--- a/core/switch_core/db/stores/retention_store.py+++ b/core/switch_core/db/stores/retention_store.py@@ -12,6 +12,7 @@ from datetime import datetimefrom typing import Any, castfrom sqlalchemy import CursorResult, Result, delete, func, select, text+from sqlalchemy.dialects.postgresql import insertfrom sqlalchemy.ext.asyncio import AsyncSessionfrom switch_core.db.models import (@@ -60,20 +61,29 @@ class RetentionStore:async def set_policy(self, session: AsyncSession, *, message_retention_days: int, user_id: str) -> TenantRetentionPolicy:- policy = await self.get_policy(session)- if policy is None:- policy = TenantRetentionPolicy(+ """Set the window, creating the policy or replacing it in one statement,+ so two people saving a first policy at once do not collide."""+ statement = (+ insert(TenantRetentionPolicy)+ .values(tenant_id=require_tenant_id(),message_retention_days=message_retention_days,updated_by_user_id=user_id,)- session.add(policy)- else:- policy.message_retention_days = message_retention_days- policy.updated_by_user_id = user_id- await session.flush()- await session.refresh(policy)- return policy+ .on_conflict_do_update(+ index_elements=[TenantRetentionPolicy.tenant_id],+ set_={+ "message_retention_days": message_retention_days,+ "updated_by_user_id": user_id,+ "updated_at": func.now(),+ },+ )+ .returning(TenantRetentionPolicy)+ )+ result = await session.execute(+ statement, execution_options={"populate_existing": True}+ )+ return result.scalar_one()async def clear_policy(self, session: AsyncSession) -> bool:"""Remove the policy, so messages are kept forever. Whether there was one."""diff --git a/core/switch_core/main.py b/core/switch_core/main.pyindex 8b96dc6dd..41de504fe 100644--- a/core/switch_core/main.py+++ b/core/switch_core/main.py@@ -211,6 +211,7 @@ from switch_core.process_lease import ProcessLeasefrom switch_core.provisioning import Provisioningfrom switch_core.provisioning.postgres import PostgresProvisioningfrom switch_core.room_changes import COALESCE_SECONDS, RoomChanges+from switch_core.retention.service import retention_loopfrom switch_core.room_service import RoomServicefrom switch_core.session_activity.listener import AgentSessionActivityListenerfrom switch_core.session_activity.maintenance import (@@ -1521,6 +1522,7 @@ async def run(config: SwitchConfig, process_id: str) -> None:if management is not Noneelse None)+ data_retention_task = asyncio.create_task(retention_loop(session_factory))# Only when telemetry is on: the chart tells a customer that off# means nothing is collected, and the fan-out is not free.snapshot_task = (@@ -1542,6 +1544,7 @@ async def run(config: SwitchConfig, process_id: str) -> None:router.close()connection_sweep_task.cancel()service_token_task.cancel()+ data_retention_task.cancel()if snapshot_task is not None:snapshot_task.cancel()shared_connections_task.cancel()diff --git a/core/switch_core/migrations/versions/d8a4e2c61f07_data_retention.py b/core/switch_core/migrations/versions/d8a4e2c61f07_data_retention.pyindex 9e9451a42..8a038330b 100644--- a/core/switch_core/migrations/versions/d8a4e2c61f07_data_retention.py+++ b/core/switch_core/migrations/versions/d8a4e2c61f07_data_retention.py@@ -7,7 +7,23 @@ so nothing changes for existing tenants when this runs.``rooms.seq_floor`` records the highest live ``seq`` retention has deletedfrom a room, so numbering continues above it when a room's newest messagesare among those deleted. It starts at 0, which changes nothing for a room-retention has never touched.+retention has never touched. A downgrade drops it, so a room retention has+emptied numbers from 1 again afterwards; cursors past that point then skip+new messages until they are reset.++Two indexes serve retention's hourly pass, which runs for every tenant:+``ix_message_attachments_tenant_uri`` answers "does any attachment still name+this file" for the file sweep, and ``ix_bridge_message_map_tenant_event``+finds the platform mappings of deleted messages.++**Not `CONCURRENTLY`**, for the reason ``b8f2d0c41e57`` gives: each holds a+lock on its table until the migration commits. A deployment with large+tables can build them by hand first, and this becomes a no-op for them:++ CREATE INDEX CONCURRENTLY ix_message_attachments_tenant_uri+ ON message_attachments (tenant_id, uri);+ CREATE INDEX CONCURRENTLY ix_bridge_message_map_tenant_event+ ON bridge_message_map (tenant_id, transport_event_id);The row-level-security DDL is a verbatim copy of ``switch_core/db/rls_ddl.py``as it stood when this migration was written, copied rather than imported for@@ -41,7 +57,20 @@ _CREATE_POLICY = ()+_INDEXES = (+ ("ix_message_attachments_tenant_uri", "message_attachments", "tenant_id, uri"),+ (+ "ix_bridge_message_map_tenant_event",+ "bridge_message_map",+ "tenant_id, transport_event_id",+ ),+)++def upgrade() -> None:+ # `IF NOT EXISTS` so building them concurrently ahead of time works.+ for name, table, columns in _INDEXES:+ op.execute(f"CREATE INDEX IF NOT EXISTS {name} ON {table} ({columns})")op.add_column("rooms",sa.Column(@@ -76,6 +105,8 @@ def upgrade() -> None:def downgrade() -> None:+ for name, _, _ in _INDEXES:+ op.execute(f"DROP INDEX IF EXISTS {name}")op.execute(f'DROP POLICY IF EXISTS {POLICY_NAME} ON "tenant_retention_policies"')op.drop_table("tenant_retention_policies")op.drop_column("rooms", "seq_floor")diff --git a/core/switch_core/retention/service.py b/core/switch_core/retention/service.pyindex eba84211b..cc192b9c5 100644--- a/core/switch_core/retention/service.py+++ b/core/switch_core/retention/service.py@@ -1,7 +1,8 @@"""Data retention: deleting what a workspace no longer keeps.-One pass over the bound tenant does three things, each in its own short-transactions so a large backlog never holds a long lock:+A pass runs hourly as a task of its own (`retention_loop`). Over each tenant+it does three things, each in its own short transactions so a large backlog+never holds a long lock, and each whether or not the others failed:1. If the workspace has a retention policy, deletes room messages older thanits window, from every room, archived ones included. Their attachment rows@@ -18,9 +19,11 @@ says why, and what retention does not yet cover.from __future__ import annotations+import asyncioimport logging+import timefrom dataclasses import dataclass-from datetime import datetime, timedelta+from datetime import UTC, datetime, timedeltafrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker@@ -28,6 +31,8 @@ from switch_core.db.models import require_tenant_idfrom switch_core.db.session_scope import tenant_sessionfrom switch_core.db.stores.message_store import MessageStorefrom switch_core.db.stores.retention_store import RetentionStore+from switch_core.db.tenant_lookup import all_tenant_ids+from switch_core.tenant_context import no_tenant, tenant_scopelogger = logging.getLogger(__name__)@@ -39,10 +44,12 @@ SETTLED_GRACE = timedelta(days=30)MEDIA_GRACE = timedelta(days=1)MESSAGE_BATCH = 1000MEDIA_BATCH = 100-#: Batches of each kind per pass. A backlog larger than this (the first pass-#: after a short window is set on a long-lived workspace) is finished by the-#: passes after it rather than by one that runs for an hour.-MAX_BATCHES = 50+#: How long one tenant's pass may spend deleting before it stops and leaves+#: the rest to the next pass, so one workspace with a large backlog (the first+#: pass after a short window is set on a long-lived workspace) does not hold+#: up every other workspace's.+TENANT_BUDGET = timedelta(minutes=2)+PASS_EVERY_SECONDS = 3600.0@dataclass(frozen=True)@@ -53,6 +60,7 @@ class RetentionPass:invitations: intinstall_states: intbacklog: bool+ failed: tuple[str, ...]class RetentionService:@@ -61,31 +69,61 @@ class RetentionService:self._messages = MessageStore()self._retention = RetentionStore()- async def apply(self, now: datetime) -> RetentionPass:- """One retention pass over the bound tenant."""- tenant_id = require_tenant_id()- async with tenant_session(self._sessions, tenant_id) as db:- policy = await self._retention.get_policy(db)- days = None if policy is None else policy.message_retention_days-- messages, message_backlog = (- (0, False)- if days is None- else await self._delete_messages(tenant_id, now - timedelta(days=days))- )- media, media_backlog = await self._delete_media(tenant_id, now - MEDIA_GRACE)+ async def apply(self, now: datetime, budget: timedelta) -> RetentionPass:+ """One retention pass over the bound tenant.- settled_before = now - SETTLED_GRACE- async with tenant_session(self._sessions, tenant_id) as db, db.begin():- approvals = await self._retention.delete_settled_approvals(- db, settled_before+ Each step runs whether or not the one before it failed: a message+ deletion that keeps failing must not also stop files and leftovers+ being cleaned up. A failure is logged with its traceback and named in+ the result.+ """+ tenant_id = require_tenant_id()+ deadline = time.monotonic() + budget.total_seconds()+ failed: list[str] = []++ messages, message_backlog = 0, False+ try:+ async with tenant_session(self._sessions, tenant_id) as db:+ policy = await self._retention.get_policy(db)+ if policy is not None:+ messages, message_backlog = await self._delete_messages(+ tenant_id,+ now - timedelta(days=policy.message_retention_days),+ deadline,+ )+ except Exception:+ logger.exception(+ "Retention could not delete messages in tenant %s", tenant_id)- invitations = await self._retention.delete_lapsed_invitations(- db, settled_before+ failed.append("messages")++ media, media_backlog = 0, False+ try:+ media, media_backlog = await self._delete_media(+ tenant_id, now - MEDIA_GRACE, deadline)- install_states = await self._retention.delete_expired_install_states(- db, settled_before+ except Exception:+ logger.exception("Retention could not sweep files in tenant %s", tenant_id)+ failed.append("media")++ approvals = invitations = install_states = 0+ settled_before = now - SETTLED_GRACE+ try:+ async with tenant_session(self._sessions, tenant_id) as db, db.begin():+ approvals = await self._retention.delete_settled_approvals(+ db, settled_before+ )+ invitations = await self._retention.delete_lapsed_invitations(+ db, settled_before+ )+ install_states = await self._retention.delete_expired_install_states(+ db, settled_before+ )+ except Exception:+ logger.exception(+ "Retention could not prune leftover records in tenant %s", tenant_id)+ failed.append("leftovers")result = RetentionPass(messages=messages,@@ -94,11 +132,12 @@ class RetentionService:invitations=invitations,install_states=install_states,backlog=message_backlog or media_backlog,+ failed=tuple(failed),)if result.backlog:logger.warning(- "Retention in tenant %s stopped at its per-pass limit with more "- "to delete; the next pass continues (%s)",+ "Retention in tenant %s ran out of time with more to delete; "+ "the next pass continues (%s)",tenant_id,result,)@@ -107,10 +146,10 @@ class RetentionService:return resultasync def _delete_messages(- self, tenant_id: str, cutoff: datetime+ self, tenant_id: str, cutoff: datetime, deadline: float) -> tuple[int, bool]:deleted = 0- for _ in range(MAX_BATCHES):+ while True:async with tenant_session(self._sessions, tenant_id) as db, db.begin():event_ids = await self._messages.delete_sent_before(db, cutoff, limit=MESSAGE_BATCH@@ -119,13 +158,14 @@ class RetentionService:deleted += len(event_ids)if len(event_ids) < MESSAGE_BATCH:return deleted, False- return deleted, True+ if time.monotonic() >= deadline:+ return deleted, Trueasync def _delete_media(- self, tenant_id: str, created_before: datetime+ self, tenant_id: str, created_before: datetime, deadline: float) -> tuple[int, bool]:deleted = 0- for _ in range(MAX_BATCHES):+ while True:async with tenant_session(self._sessions, tenant_id) as db, db.begin():count = await self._retention.delete_unreferenced_media(db, created_before=created_before, limit=MEDIA_BATCH@@ -133,4 +173,35 @@ class RetentionService:deleted += countif count < MEDIA_BATCH:return deleted, False- return deleted, True+ if time.monotonic() >= deadline:+ return deleted, True+++async def retain_once(+ session_factory: async_sessionmaker[AsyncSession], now: datetime+) -> None:+ """One pass over every tenant, each bound before it is touched."""+ service = RetentionService(session_factory)+ for tenant_id in await all_tenant_ids(session_factory):+ with tenant_scope(tenant_id):+ try:+ await service.apply(now, TENANT_BUDGET)+ except Exception:+ # One tenant's failure must not stop the others' retention.+ logger.exception("Data retention failed for tenant %s", tenant_id)+++async def retention_loop(session_factory: async_sessionmaker[AsyncSession]) -> None:+ """Retention at start and hourly, as a task of its own.++ Not in the session-activity upkeep loop, which also expires approval+ requests every few seconds: a long retention pass there would hold up+ what unblocks a stuck session.+ """+ with no_tenant():+ while True:+ try:+ await retain_once(session_factory, datetime.now(UTC))+ except Exception:+ logger.exception("Data retention could not list tenants")+ await asyncio.sleep(PASS_EVERY_SECONDS)diff --git a/core/switch_core/session_activity/maintenance.py b/core/switch_core/session_activity/maintenance.pyindex bf920729d..1a62bbaf3 100644--- a/core/switch_core/session_activity/maintenance.py+++ b/core/switch_core/session_activity/maintenance.py@@ -3,8 +3,7 @@Expiry is what unblocks a session nobody answers, so it runs often; eachexpired request is announced like any other change and pushed to the agent.Activity lines exist for platforms to show what is happening now, so they are-kept for a bounded window and pruned rarely. Data retention-(`retention/service.py`) rides the same hourly prune.+kept for a bounded window and pruned rarely.Tenants are worked through one at a time, each bound before it is touched,like every other background task that spans them.@@ -15,13 +14,12 @@ from __future__ import annotationsimport asyncioimport contextlibimport logging-from datetime import UTC, datetime, timedelta+from datetime import timedeltafrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmakerfrom switch_core.db.tenant_lookup import all_tenant_idsfrom switch_core.ownership import Fence, OwnedKind, Ownership-from switch_core.retention.service import RetentionServicefrom switch_core.session_activity.service import AgentSessionActivityServicefrom switch_core.tenant_context import no_tenant, tenant_scope@@ -36,18 +34,10 @@ ACTIVITY_RETENTION = timedelta(days=7)async def maintain_once(session_factory: async_sessionmaker[AsyncSession], *, prune: bool) -> None:- """One pass over every tenant: expire overdue requests, and if asked, prune- activity and apply data retention."""+ """One pass over every tenant: expire overdue requests, and prune if asked."""service = AgentSessionActivityService(session_factory)- retention = RetentionService(session_factory)for tenant_id in await all_tenant_ids(session_factory):with tenant_scope(tenant_id):- if prune:- try:- await retention.apply(datetime.now(UTC))- except Exception:- # Retention failing must not stop this tenant's requests expiring.- logger.exception("Data retention failed for tenant %s", tenant_id)try:expired = await service.expire_due()if expired:diff --git a/core/tests/switch_core/db/test_tenant_exemption_allowlist.py b/core/tests/switch_core/db/test_tenant_exemption_allowlist.pyindex b66d3d7fa..a7cddd492 100644--- a/core/tests/switch_core/db/test_tenant_exemption_allowlist.py+++ b/core/tests/switch_core/db/test_tenant_exemption_allowlist.py@@ -71,6 +71,10 @@ _ALLOWED_MODULES = {# workspace: enumerate, then read each one's connections under its own# policy, the same shape as service-token upkeep above."switch_core.gateway.connector_apps",+ # Data retention deletes what each workspace no longer keeps, hourly:+ # enumerate, then bind each tenant and work under its own policy, the same+ # shape as session-activity upkeep.+ "switch_core.retention.service",# The runtime-state sweep reads every tenant's stale rows, one tenant at a# time."switch_core.bridges.agent.protocol.agent_core",diff --git a/core/tests/switch_core/retention/test_service.py b/core/tests/switch_core/retention/test_service.pyindex aaed49ea0..548bbbaea 100644--- a/core/tests/switch_core/retention/test_service.py+++ b/core/tests/switch_core/retention/test_service.py@@ -8,9 +8,11 @@ that one workspace's window never reaches another's rows.from __future__ import annotations+import asyncioimport uuidfrom datetime import UTC, datetime, timedelta+import pytestfrom sqlalchemy import func, selectfrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker@@ -34,10 +36,12 @@ from switch_core.db.models import (from switch_core.db.session_scope import tenant_sessionfrom switch_core.db.stores.message_store import MessageStorefrom switch_core.db.stores.retention_store import RetentionStore+from switch_core.retention import service as retention_servicefrom switch_core.retention.service import RetentionServicefrom switch_core.tenant_context import tenant_scopeNOW = datetime(2026, 10, 6, 12, 0, tzinfo=UTC)+BUDGET = timedelta(minutes=5)OTHER_TENANT = "retention-other-tenant"@@ -134,7 +138,7 @@ class TestMessages:await _message(session, room, "$ancient", age_days=3000)await session.commit()- result = await RetentionService(session_factory).apply(NOW)+ result = await RetentionService(session_factory).apply(NOW, BUDGET)async with session_factory() as session:assert await _event_ids(session) == {"$ancient"}@@ -152,7 +156,7 @@ class TestMessages:await session.commit()await _set_policy(session_factory, 30, TENANT_ZERO_ID)- result = await RetentionService(session_factory).apply(NOW)+ result = await RetentionService(session_factory).apply(NOW, BUDGET)async with session_factory() as session:assert await _event_ids(session) == {"$new-active"}@@ -194,7 +198,7 @@ class TestMessages:await session.commit()await _set_policy(session_factory, 30, TENANT_ZERO_ID)- await RetentionService(session_factory).apply(NOW)+ await RetentionService(session_factory).apply(NOW, BUDGET)async with session_factory() as session:attachments = await session.scalar(@@ -225,9 +229,9 @@ class TestMessages:await _set_policy(session_factory, 30, OTHER_TENANT)with tenant_scope(OTHER_TENANT):- await RetentionService(session_factory).apply(NOW)+ await RetentionService(session_factory).apply(NOW, BUDGET)with tenant_scope(TENANT_ZERO_ID):- await RetentionService(session_factory).apply(NOW)+ await RetentionService(session_factory).apply(NOW, BUDGET)async with session_factory() as session:assert await _event_ids(session) == {"$zero-old"}@@ -246,7 +250,7 @@ class TestNumbering:head_before = await store.head_seq(session, room.id)await _set_policy(session_factory, 30, TENANT_ZERO_ID)- await RetentionService(session_factory).apply(NOW)+ await RetentionService(session_factory).apply(NOW, BUDGET)async with session_factory() as session:head_after = await store.head_seq(session, room.id)@@ -278,7 +282,7 @@ class TestNumbering:await session.commit()await _set_policy(session_factory, 30, TENANT_ZERO_ID)- await RetentionService(session_factory).apply(NOW)+ await RetentionService(session_factory).apply(NOW, BUDGET)async with session_factory() as session:refreshed = await session.get(Room, room.id)@@ -305,7 +309,7 @@ class TestMedia:await _blob(session, "switch-media://just-uploaded", age_days=0.1)await session.commit()- result = await RetentionService(session_factory).apply(NOW)+ result = await RetentionService(session_factory).apply(NOW, BUDGET)async with session_factory() as session:uris = set((await session.execute(select(MediaBlob.uri))).scalars())@@ -327,13 +331,13 @@ class TestMedia:await session.commit()await _set_policy(session_factory, 30, TENANT_ZERO_ID)- await RetentionService(session_factory).apply(NOW)+ await RetentionService(session_factory).apply(NOW, BUDGET)async with session_factory() as session:assert (await session.scalar(select(func.count()).select_from(MediaBlob)) == 1)- await RetentionService(session_factory).apply(NOW + timedelta(days=21))+ await RetentionService(session_factory).apply(NOW + timedelta(days=21), BUDGET)async with session_factory() as session:assert (await session.scalar(select(func.count()).select_from(MediaBlob)) == 0@@ -398,7 +402,7 @@ class TestLeftovers:)await session.commit()- result = await RetentionService(session_factory).apply(NOW)+ result = await RetentionService(session_factory).apply(NOW, BUDGET)async with session_factory() as session:left = set(@@ -437,7 +441,7 @@ class TestLeftovers:)await session.commit()- result = await RetentionService(session_factory).apply(NOW)+ result = await RetentionService(session_factory).apply(NOW, BUDGET)async with session_factory() as session:hashes = set(@@ -449,3 +453,67 @@ class TestLeftovers:assert hashes == {"expired-recent", "live"}assert states == 1assert (result.invitations, result.install_states) == (2, 1)+++class TestPass:+ async def test_a_failing_step_does_not_stop_the_others(+ self,+ session_factory: async_sessionmaker[AsyncSession],+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ async with session_factory() as session:+ await _blob(session, "switch-media://orphan", age_days=10)+ await session.commit()+ await _set_policy(session_factory, 30, TENANT_ZERO_ID)++ async def broken(*args: object, **kwargs: object) -> list[str]:+ raise RuntimeError("deleting messages is broken")++ monkeypatch.setattr(MessageStore, "delete_sent_before", broken)+ result = await RetentionService(session_factory).apply(NOW, BUDGET)++ assert result.failed == ("messages",)+ assert result.media == 1++ async def test_a_pass_out_of_time_stops_after_a_batch_and_says_so(+ self,+ session_factory: async_sessionmaker[AsyncSession],+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ monkeypatch.setattr(retention_service, "MESSAGE_BATCH", 2)+ async with session_factory() as session:+ room = await _room(session, "backlog")+ for index in range(5):+ await _message(session, room, f"$old{index}", age_days=60)+ await session.commit()+ await _set_policy(session_factory, 30, TENANT_ZERO_ID)++ first = await RetentionService(session_factory).apply(NOW, timedelta(0))+ rest = await RetentionService(session_factory).apply(NOW, BUDGET)++ assert (first.messages, first.backlog) == (2, True)+ assert (rest.messages, rest.backlog) == (3, False)+++class TestPolicy:+ async def test_two_first_saves_at_once_both_succeed(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ user = await _user(session, "racer")+ await session.commit()+ user_id = user.id++ async def save(days: int) -> None:+ async with session_factory() as session:+ await RetentionStore().set_policy(+ session, message_retention_days=days, user_id=user_id+ )+ await session.commit()++ await asyncio.gather(save(30), save(90))++ async with session_factory() as session:+ policy = await RetentionStore().get_policy(session)+ assert policy is not None+ assert policy.message_retention_days in (30, 90)diff --git a/docs/design/data-retention.md b/docs/design/data-retention.mdindex 5703ba4a6..9366e3601 100644--- a/docs/design/data-retention.md+++ b/docs/design/data-retention.md@@ -25,13 +25,17 @@ which chat-platform post carried each message.- **Audit.** Setting and clearing the window are recorded as`retention_policy.set` and `retention_policy.cleared`, with the previous andnew values.-- **Enforcement.** A pass runs hourly per workspace, inside the- session-activity maintenance loop (`session_activity/maintenance.py` calls- `retention/service.py`). A change therefore takes effect within the hour, not- in the request that made it. Deletion runs in batches of 1000 messages, each- in its own transaction, and stops at 50 batches per pass. A larger backlog,- such as the first pass after a short window is set on an old workspace, is- finished by the passes after it. Hitting the limit logs a warning.+- **Enforcement.** A pass runs at startup and then hourly, as a background+ task of its own (`retention_loop` in `retention/service.py`). It is kept out+ of the session-activity upkeep loop, which expires approval requests every+ few seconds, so a long pass cannot hold up what unblocks a stuck session. A+ change therefore takes effect within the hour, not in the request that made+ it. Deletion runs in batches of 1000 messages, each in its own transaction,+ and each workspace gets two minutes per pass. A larger backlog, such as the+ first pass after a short window is set on an old workspace, is finished by+ the passes after it, and running out of time logs a warning. Messages, files+ and leftover records are separate steps: one failing is logged and does not+ skip the others.### Message numbering survives deletion@@ -51,6 +55,16 @@ Other effects of deleting old messages, all already handled:- Posting a new reply into a thread whose root was deleted is refused with"thread_id not found". That is a visible failure, not a silent one.+One effect is not signalled. A message an agent has not yet received (its+cursor is behind it) and that ages out is simply never delivered, with no+"history lost" notice. In practice this needs an agent to be days behind a+room. A window shorter than the longest an agent may be offline is a choice+the workspace makes with that trade-off.++A downgrade past this migration drops `seq_floor`, so a room retention has+emptied numbers from 1 again, and cursors past that point skip new messages+until they are reset.+### Orphaned files are sweptAttachment bytes live in `media_blobs`, which deliberately has no foreign key todiff --git a/gateway/src/pages/workspace/RetentionSection.test.tsx b/gateway/src/pages/workspace/RetentionSection.test.tsxindex 90dd5adda..6a9ef648a 100644--- a/gateway/src/pages/workspace/RetentionSection.test.tsx+++ b/gateway/src/pages/workspace/RetentionSection.test.tsx@@ -41,6 +41,18 @@ describe("describeRetention", () => {});describe("RetentionSection", () => {+ it("shows no setting at all when the policy cannot be read", async () => {+ vi.stubGlobal(+ "fetch",+ vi.fn(async () => new Response(JSON.stringify({ detail: "boom" }), { status: 500 })),+ );+ render(<RetentionSection tenantId="t1" />);++ await screen.findByText("boom");+ expect(screen.queryByText(/kept forever/)).toBeNull();+ expect(screen.queryByRole("button", { name: "Save" })).toBeNull();+ });+afterEach(() => {cleanup();vi.unstubAllGlobals();diff --git a/gateway/src/pages/workspace/RetentionSection.tsx b/gateway/src/pages/workspace/RetentionSection.tsxindex 90db0c865..9315d522b 100644--- a/gateway/src/pages/workspace/RetentionSection.tsx+++ b/gateway/src/pages/workspace/RetentionSection.tsx@@ -120,13 +120,14 @@ export default function RetentionSection({ tenantId }: { tenantId: string }) {)}{loading ? (<CircularProgress />- ) : (+ ) : error ? null : (<Paper variant="outlined" sx={{ p: 2 }}><Stack spacing={2}><Typography variant="body2" sx={{ color: "text.secondary" }}>{describeRetention(current)} This applies to every room in the workspace,- archived rooms included. Attachments are deleted with their messages. Copies- already posted to Slack or other connected chat apps are not affected.+ archived rooms included. A message's files are deleted about a day after the+ last message using them. Copies already posted to Slack or other connected chat+ apps are not affected.</Typography><RadioGroupvalue={mode}@@ -191,7 +192,7 @@ export default function RetentionSection({ tenantId }: { tenantId: string }) {<DialogContentText>{confirm?.count === 0? "No messages are old enough to be deleted yet."- : `${confirm?.count.toLocaleString()} ${confirm?.count === 1 ? "message" : "messages"} and their attachments will be permanently deleted within the next hour.`}{" "}+ : `${confirm?.count.toLocaleString()} ${confirm?.count === 1 ? "message" : "messages"} will be permanently deleted over the coming hours, starting within the hour. Their files follow about a day later.`}{" "}From then on, messages are deleted as they pass {confirm?.days} days old. Deletedmessages cannot be recovered.</DialogContentText>