Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Adapt data retention and erasure to current main
Petr Bauch93f8adc7bd5e
diff --git a/CLAUDE.md b/CLAUDE.mdindex b9cf6da00..5cfde7509 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -223,8 +223,8 @@ Tests live in `core/tests/switch_core/` mirroring the module structure. Uses pytrotation runbook- `docs/design/data-retention.md` — what Switch deletes: the per-workspacemessage-retention window, the orphaned-file sweep, the leftover records- pruned after a grace period, and a GDPR design note for what is not built- (export, workspace deletion, audit-log horizon) and how erasing a person works+ pruned after a grace period, how erasing a person works, and a GDPR design+ note for what is not built (export, audit-log horizon)- `docs/old/rds-migration.md` — moving a deployment's Postgres to RDS: theproposal and the cutover runbookdiff --git a/core/switch_core/accounts/deletion.py b/core/switch_core/accounts/deletion.pyindex 95f13fcd7..09be7dcb1 100644--- a/core/switch_core/accounts/deletion.py+++ b/core/switch_core/accounts/deletion.py@@ -161,6 +161,7 @@ USER_REFERENCES: dict[tuple[str, str], Handling] = {("organization_app_credentials", "updated_by"): "database clears",("organization_service_settings", "updated_by"): "database clears",("tenant_retention_policies", "updated_by_user_id"): "database clears",+ ("person_erasures", "requested_by_user_id"): "database clears",}diff --git a/core/switch_core/accounts/workspace_deletion.py b/core/switch_core/accounts/workspace_deletion.pyindex 7b2893724..52a5b88ff 100644--- a/core/switch_core/accounts/workspace_deletion.py+++ b/core/switch_core/accounts/workspace_deletion.py@@ -143,6 +143,7 @@ WORKSPACE_TABLES: dict[str, Handling] = {"tenant_data_moves": "deleted","tenant_join_domains": "deleted","tenant_retention_policies": "deleted",+ "person_erasures": "deleted","tenant_usage": "deleted","usage_budgets": "deleted","workspace_service_settings": "deleted",diff --git a/core/switch_core/db/stores/erasure_store.py b/core/switch_core/db/stores/erasure_store.pyindex e16564cb7..c0bba8907 100644--- a/core/switch_core/db/stores/erasure_store.py+++ b/core/switch_core/db/stores/erasure_store.py@@ -32,14 +32,13 @@ from sqlalchemy.dialects.postgresql import arrayfrom sqlalchemy.ext.asyncio import AsyncSessionfrom switch_core.db.models import (+ AgentEventRow,ApprovalRequest,Client,ClientRoom,CollaborationBridge,ExternalUser,ExternalUserClaim,- HostedCutoverItem,- HostedWakeMailbox,Message,PersonErasure,Room,@@ -270,7 +269,7 @@ class ErasureStore:) -> PersonErasure:"""Queue the erasure of these identities, or raise.- Locks the tenant's erasure queue for the transaction, so two owners+ Locks the tenant's erasure queue for the transaction, so two adminsasking at once cannot both pass the "nothing already queued" check."""tenant_id = require_tenant_id()@@ -413,24 +412,18 @@ class ErasureStore:)return result.scalar_one()- async def delete_hosted_copies(+ async def delete_agent_copies(self, session: AsyncSession, transport_event_ids: Collection[str]) -> None:- """Delete the copies of these (deleted) messages held for hosted agents:- deliveries not yet admitted, and items a cutover would import."""+ """Delete the copies of these (deleted) messages stored as events for+ agents (`agent_events`), which would otherwise outlive them until the+ event retention sweep."""if not transport_event_ids:return- ids = list(transport_event_ids)await session.execute(- delete(HostedWakeMailbox).where(- HostedWakeMailbox.tenant_id == require_tenant_id(),- HostedWakeMailbox.message_id.in_(ids),- )- )- await session.execute(- delete(HostedCutoverItem).where(- HostedCutoverItem.tenant_id == require_tenant_id(),- HostedCutoverItem.message_id.in_(ids),+ delete(AgentEventRow).where(+ AgentEventRow.tenant_id == require_tenant_id(),+ AgentEventRow.message_key.in_(list(transport_event_ids)),))diff --git a/core/switch_core/db/stores/retention_store.py b/core/switch_core/db/stores/retention_store.pyindex 8101096f4..3e6e61178 100644--- a/core/switch_core/db/stores/retention_store.py+++ b/core/switch_core/db/stores/retention_store.py@@ -25,21 +25,17 @@ from switch_core.db.models import (require_tenant_id,)-#: A stored file no attachment names and no live hosted-cutover import is-#: waiting to carry, for a `media_blobs` row aliased `b`. Shared with erasing a+#: A stored file no attachment names and no staged chat upload is still+#: waiting to send, for a `media_blobs` row aliased `b`. Shared with erasing a#: person, which deletes files by the same rule without the age condition.UNREFERENCED_BLOB = """NOT EXISTS (SELECT 1 FROM message_attachments aWHERE a.tenant_id = b.tenant_id AND a.uri = b.uri)AND NOT EXISTS (- SELECT 1 FROM hosted_cutover_items i- JOIN hosted_launches l ON l.tenant_id = i.tenant_id AND l.id = i.launch_id- AND l.state NOT IN ('deleting', 'deleted')- CROSS JOIN LATERAL jsonb_array_elements(- COALESCE(i.payload->'payload'->'attachments', '[]'::jsonb)) f- WHERE i.tenant_id = b.tenant_id AND i.disposition = 'import'- AND f->>'mxc' = b.uri)+ SELECT 1 FROM chat_operations o+ WHERE o.tenant_id = b.tenant_id AND o.kind = 'upload'+ AND o.result->>'uri' = b.uri)"""# A blob is written before the message that carries it, in a separatediff --git a/core/switch_core/gateway/tenants.py b/core/switch_core/gateway/tenants.pyindex ad309a510..9f84ce84c 100644--- a/core/switch_core/gateway/tenants.py+++ b/core/switch_core/gateway/tenants.py@@ -927,6 +927,7 @@ def _erasure_detail(erasure: PersonErasure) -> ErasureDetail:completed_at=erasure.completed_at,)+async def _require_admin_membership(session: AsyncSession, user_store: UserStore, tenant_id: str, user: User) -> None:@@ -939,7 +940,6 @@ async def _require_admin_membership()-@router.get("/tenants/{tenant_id}/people")async def list_people(tenant_id: str,diff --git a/core/switch_core/main.py b/core/switch_core/main.pyindex f60807b44..079a5429e 100644--- a/core/switch_core/main.py+++ b/core/switch_core/main.py@@ -210,10 +210,9 @@ from switch_core.process_identity import new_process_idfrom switch_core.process_lease import ProcessLeasefrom switch_core.provisioning import Provisioningfrom switch_core.provisioning.postgres import PostgresProvisioning-from switch_core.room_changes import COALESCE_SECONDS, RoomChanges-from switch_core.retention.service import retention_loopfrom switch_core.retention.erasure import ErasureService, erasure_loopfrom switch_core.retention.service import retention_loop+from switch_core.room_changes import COALESCE_SECONDS, RoomChangesfrom switch_core.room_service import RoomServicefrom switch_core.session_activity.listener import AgentSessionActivityListenerfrom switch_core.session_activity.maintenance import (diff --git a/core/switch_core/retention/erasure.py b/core/switch_core/retention/erasure.pyindex b307252d1..4917b8cb0 100644--- a/core/switch_core/retention/erasure.py+++ b/core/switch_core/retention/erasure.py@@ -1,6 +1,6 @@"""Erasing a person from a workspace.-A workspace owner queues a request naming one or more platform identities+A workspace admin queues a request naming one or more platform identities(`ErasureStore.queue`), and any former participants: people from a chat appsince disconnected, whose identity rows went with it. For those only theirmessages, files, approval answers and direct-room names are left to erase@@ -8,7 +8,7 @@ messages, files, approval answers and direct-room names are left to erase1. deletes every message the identity sent, in every room, archived onesincluded, in batches with their attachments, bridge post mappings and the- copies held for hosted agents;+ copies stored as events for agents;2. stops the client that stood in for them in rooms;3. in one transaction, scrubs their name and words from approval answers andtheir name from direct rooms' names, deletes their identity row (its@@ -252,7 +252,7 @@ class ErasureService:db, transport_user_id=transport_user_id, limit=MESSAGE_BATCH)await self._retention.delete_bridge_mappings(db, deleted.event_ids)- await self._erasures.delete_hosted_copies(db, deleted.event_ids)+ await self._erasures.delete_agent_copies(db, deleted.event_ids)await self._erasures.add_progress(db, erasure_id, messages=len(deleted.event_ids), files=0, identities=0)diff --git a/core/tests/switch_core/gateway/test_tenant_api_routes.py b/core/tests/switch_core/gateway/test_tenant_api_routes.pyindex 0d37f6769..a28036324 100644--- a/core/tests/switch_core/gateway/test_tenant_api_routes.py+++ b/core/tests/switch_core/gateway/test_tenant_api_routes.py@@ -49,7 +49,6 @@ from switch_core.db.models import (Message,Organization,OrganizationMember,- ProviderConnection,Room,ServiceTokenIssuance,SignInFlow,@@ -2784,7 +2783,7 @@ class TestRetentionRoutes:self, session_factory: async_sessionmaker[AsyncSession], days: int) -> None:_, token = await self._member(- session_factory, f"retention-range-{days}", "owner"+ session_factory, f"retention-range-{days}", "admin")async with _client(_app(session_factory), token) as client:@@ -2797,7 +2796,7 @@ class TestRetentionRoutes:async def test_another_workspace_in_the_path_is_refused(self, session_factory: async_sessionmaker[AsyncSession]) -> None:- _, token = await self._member(session_factory, "retention-cross", "owner")+ _, token = await self._member(session_factory, "retention-cross", "admin")await _make_tenant(session_factory, TENANT_B)async with _client(_app(session_factory), token) as client:@@ -2888,11 +2887,11 @@ class TestErasureRoutes:await session.commit()return person.id- async def test_an_owner_lists_people_queues_an_erasure_and_sees_it(+ async def test_an_admin_lists_people_queues_an_erasure_and_sees_it(self, session_factory: async_sessionmaker[AsyncSession]) -> None:await _make_tenant(session_factory, TENANT_A)- token = await self._member(session_factory, "erase-owner", "owner")+ token = await self._member(session_factory, "erase-admin", "admin")person_id = await self._person(session_factory, "ana")async with _client(_app(session_factory), token) as client:@@ -2935,7 +2934,7 @@ class TestErasureRoutes:self, session_factory: async_sessionmaker[AsyncSession]) -> None:await _make_tenant(session_factory, TENANT_A)- token = await self._member(session_factory, "erase-former", "owner")+ token = await self._member(session_factory, "erase-former", "admin")sender_id = f"@switch-slack-{uuid.uuid4()}-dee:test"async with tenant_session(session_factory, TENANT_A) as session:room = Room(@@ -2977,29 +2976,39 @@ class TestErasureRoutes:assert queued.status_code == 202assert queued.json()["identities"] == 1- async def test_an_admin_cannot_erase_or_list_people(+ async def test_a_member_or_an_operator_without_admin_membership_cannot_erase(self, session_factory: async_sessionmaker[AsyncSession]) -> None:await _make_tenant(session_factory, TENANT_A)- token = await self._member(session_factory, "erase-admin", "admin")+ member = await self._member(session_factory, "erase-member", "member")+ operator_id = await _make_member(+ session_factory,+ name="erase-operator",+ tenant_id=TENANT_A,+ role="member",+ user_role="admin",+ )+ operator = create_jwt(+ operator_id, "[email protected]", "admin", _KEYRING, TENANT_A+ )person_id = await self._person(session_factory, "ana")- async with _client(_app(session_factory), token) as client:- people = await client.get(f"/tenants/{TENANT_A}/people")- queued = await client.post(- f"/tenants/{TENANT_A}/erasures",- json={"external_user_ids": [person_id]},- )-- assert people.status_code == 403- assert queued.status_code == 403+ for token in (member, operator):+ async with _client(_app(session_factory), token) as client:+ people = await client.get(f"/tenants/{TENANT_A}/people")+ queued = await client.post(+ f"/tenants/{TENANT_A}/erasures",+ json={"external_user_ids": [person_id]},+ )+ assert people.status_code == 403+ assert queued.status_code == 403async def test_a_person_from_another_workspace_is_not_found(self, session_factory: async_sessionmaker[AsyncSession]) -> None:await _make_tenant(session_factory, TENANT_A)await _make_tenant(session_factory, TENANT_B)- token = await self._member(session_factory, "erase-cross", "owner")+ token = await self._member(session_factory, "erase-cross", "admin")async with tenant_session(session_factory, TENANT_B) as session:other_client = Client(transport_user_id=f"@b-{uuid.uuid4().hex[:8]}:test",diff --git a/core/tests/switch_core/retention/test_erasure.py b/core/tests/switch_core/retention/test_erasure.pyindex d04a21b98..0f965b11e 100644--- a/core/tests/switch_core/retention/test_erasure.py+++ b/core/tests/switch_core/retention/test_erasure.py@@ -17,6 +17,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmakerfrom switch_core.db.models import (Agent,+ AgentEventRow,ApiKey,ApprovalRequest,AuditEvent,@@ -25,8 +26,6 @@ from switch_core.db.models import (CollaborationBridge,ExternalUser,ExternalUserClaim,- HostedLaunch,- HostedWakeMailbox,MediaBlob,Message,MessageAttachment,@@ -489,7 +488,7 @@ class TestErasing:assert row.answered_by == ERASED_ANSWERERassert row.answers == [{"question_id": "q1", "selected_option_ids": ["yes"]}]- async def test_their_name_leaves_direct_rooms_and_hosted_copies_go(+ async def test_their_name_leaves_direct_rooms_and_agent_copies_go(self, session_factory: async_sessionmaker[AsyncSession]) -> None:async with session_factory() as session:@@ -515,30 +514,18 @@ class TestErasing:ClientRoom(client_id=client.id, room_id=channel.id),])- message = await _say(session, dm, client, "$to-hosted")- owner = User(name="launch-owner", email="[email protected]", role="user")- session.add(owner)- await session.flush()- launch = HostedLaunch(- id=f"launch-{uuid.uuid4().hex[:8]}",- owner_id=owner.id,- name="helper",- spec={},- state="ready",- agent_id=str(uuid.uuid4()),- )- session.add(launch)- await session.flush()+ message = await _say(session, dm, client, "$to-agent")+ agent = await _agent(session)session.add(- HostedWakeMailbox(- agent_id=launch.agent_id,+ AgentEventRow(+ agent_id=agent.id,+ seq=1,room_id=dm.id,- message_id=message.transport_event_id,- launch_id=launch.id,- event={"body": "hello"},- addressed_at=datetime.now(UTC),- updated_at=datetime.now(UTC),- expires_at=datetime.now(UTC) + timedelta(hours=1),+ room_seq=message.seq,+ message_key=message.transport_event_id,+ type="message",+ payload={"body": "hello"},+ notifiable=True,))await session.commit()@@ -552,14 +539,14 @@ class TestErasing:async with session_factory() as session:dm_after = await session.get(Room, dm_id)channel_after = await session.get(Room, channel_id)- mailbox = await session.scalar(- select(func.count()).select_from(HostedWakeMailbox)+ copies = await session.scalar(+ select(func.count()).select_from(AgentEventRow))assert dm_after is not None and channel_after is not Noneassert dm_after.name == "Acme Slack: erased person / helper"assert dm_after.description == "Acme Slack DM — erased person / helper"assert channel_after.name == "Acme Slack: ana-fans"- assert mailbox == 0+ assert copies == 0class TestQueue:diff --git a/docs/design/data-retention.md b/docs/design/data-retention.mdindex 5bca26cc9..46eb9abd7 100644--- a/docs/design/data-retention.md+++ b/docs/design/data-retention.md@@ -9,7 +9,7 @@ shared picture and not from scratch.### A message-retention window per workspace-Each workspace keeps room messages forever until an owner or admin sets a+Each workspace keeps room messages forever until a workspace admin sets awindow on the Workspace page ("Data retention"). With a window of N days, roommessages older than N days are deleted from every room in the workspace,archived rooms included. Their attachments go with them, as does the record of@@ -21,7 +21,7 @@ which chat-platform post carried each message.- **API.** `GET`, `PUT` and `DELETE /tenants/{id}/retention` read, set and clearthe window. `GET /tenants/{id}/retention/preview?days=N` counts what a windowwould delete now, which the page shows before anything is saved. All four are- limited to owners and admins.+ limited to workspace admins.- **Audit.** Setting and clearing the window are recorded as`retention_policy.set` and `retention_policy.cleared`, with the previous andnew values.@@ -75,8 +75,8 @@ covers files left behind by retention and also those left behind when a room isdeleted, which nothing cleaned up before.A blob is only a candidate once it is a day old. A file is uploaded before the-message that carries it, in a separate transaction. A blob named by a pending-hosted-cutover import is never a candidate.+message that carries it, in a separate transaction. A blob named by a chat+upload that is staged but not yet sent is never a candidate.### Leftover operational records@@ -121,9 +121,11 @@ below, in the order it is likely to be needed.### Erase a person (built)-A workspace owner can erase one person from the People list on the Workspace-page, for example to answer an Art. 17 request. Admins cannot; the action-cannot be undone.+A workspace admin can erase one person from the People list on the Workspace+page, for example to answer an Art. 17 request. Because it cannot be undone,+it takes the caller's own admin membership, as deleting the workspace does:+the deployment operator's bypass and an organization admin acting as one are+refused.**Who a person is.** Everyone in a room is a chat-platform identity: an`external_users` row on one bridge, standing behind a `clients` row that is@@ -135,7 +137,7 @@ shared claimant is a hint, not proof of a shared person.**The flow.**-1. The owner picks the person and confirms by typing their name. The typed+1. The admin picks the person and confirms by typing their name. The typedname is not stored.2. `POST /tenants/{id}/erasures` queues a `person_erasures` row naming theidentities, and records `person_erasure.requested` (who asked, the internal@@ -145,9 +147,8 @@ shared claimant is a hint, not proof of a shared person.marking it running in the same statement and skipping one another processholds, and works it. For each identity it:- deletes every message it sent, in every room including archived ones, in- batches with their attachments, bridge post mappings and the copies held- for hosted agents (wake-mailbox deliveries not yet admitted, cutover- items), keeping each room's `seq_floor` and adding to the request's+ batches with their attachments, bridge post mappings and the copies stored+ as events for agents (`agent_events`), keeping each room's `seq_floor` and adding to the request'scounts in the same transaction;- stops the client that stood in for the person;- in one transaction: drops their name and the words they typed from@@ -161,14 +162,14 @@ shared claimant is a hint, not proof of a shared person.- tells the running bridge to forget the identity, so writing againprovisions a new one;- deletes the stored files that only their messages carried, with the same- hosted-cutover guard as the hourly sweep but no grace period, because+ staged-upload guard as the hourly sweep but no grace period, becausethese files are known to have been attached.4. The page polls `GET /tenants/{id}/erasures` for progress, and re-reads thepeople list once the last request finishes. On success the request records`person_erasure.completed` with its counts.5. A failure before an identity's rows are deleted marks the request `failed`with the error. It is not retried, and since the identity is still there- the owner can queue it again. A failure after (the bridge could not be+ the admin can queue it again. A failure after (the bridge could not betold, the files could not be deleted) is logged as a warning and does notfail the request: the identity is gone, so queueing again could not help.The bridge forgets on its next restart, and the hourly sweep takes the@@ -208,28 +209,28 @@ has the shape a bridge gives a person (`@switch-{platform}-{bridge id}-…`) andfor whom no client row is left. The People list shows them under "Disconnectedapp", named by the display names their messages carry, and an erasure requestnames them by that sender id (`former_sender_ids`). Erasing one deletes their-messages, files, bridge post mappings and hosted copies, scrubs their approval+messages, files, bridge post mappings and agent event copies, scrubs their approvalanswers, and replaces their names in the direct rooms among the rooms theirmessages were deleted from, since their memberships went with their client.There is no identity left to delete and no bridge to tell. Listing them scans-the tenant's messages, which is fine for an owner-only page opened rarely; an+the tenant's messages, which is fine for an admin-only page opened rarely; anindex on the sender's shape would be the fix if it is not.### Data export (not built)-An owner or admin export of everything held about one person in the workspace,+A workspace admin's export of everything held about one person in the workspace,as machine-readable JSON: account fields, platform identities, and messagesthey sent, with attachments. The same identity resolution as erasure applies.This is the natural next step, since it needs the same identity lookup aserasure (`ErasureStore.people_with_ids`).-### Deleting a workspace (not built)+### Deleting a workspace (built separately)-`tenants` has no `deleted_at` today. Deleting a workspace needs an ordered-teardown across every tenant-scoped table, because the tenant foreign keys do-not cascade. It also needs a grace period and an audit record kept outside the-tenant. This is a multi-tenancy-phase concern, noted here because it is the-largest remaining erasure path.+A workspace admin can delete a workspace once it has been emptied+(`accounts/workspace_deletion.py`): its own setup goes with it in one+transaction, including its retention policy and erasure requests, and the+deletion is recorded in the server's audit log. The workspace's own audit+history stays, which the audit-log horizon below would bound.### The audit log@@ -257,4 +258,4 @@ personal-data processing.removes the rows for the messages it deletes, but not these.- Expired controller enrolment codes. Each is tied to an `api_keys` row thatneeds to go with it.-- `tasks`, `hosted_operations` and agent-controller operation history.+- `tasks` and agent-controller operation history.diff --git a/gateway/src/pages/workspace/EraseDialog.tsx b/gateway/src/pages/workspace/EraseDialog.tsxindex 29a25480c..3089cab76 100644--- a/gateway/src/pages/workspace/EraseDialog.tsx+++ b/gateway/src/pages/workspace/EraseDialog.tsx@@ -53,7 +53,7 @@ export default function EraseDialog({ tenantId, person, people, onClose, onQueue// Reset only when a different person is chosen. The list behind `people` is// refreshed while other erasures run, and resetting on every refresh would- // undo what the owner ticked or typed just before they confirm.+ // undo what the admin ticked or typed just before they confirm.const personId = person?.id;useEffect(() => {setIncluded(new Set());diff --git a/gateway/src/pages/workspace/PeopleSection.tsx b/gateway/src/pages/workspace/PeopleSection.tsxindex 52b11a095..0d9acf850 100644--- a/gateway/src/pages/workspace/PeopleSection.tsx+++ b/gateway/src/pages/workspace/PeopleSection.tsx@@ -122,7 +122,7 @@ export default function PeopleSection({ tenantId }: { tenantId: string }) {Everyone seen in this workspace's rooms through a chat app, including apps sincedisconnected. Erasingsomeone permanently deletes every message they sent and their files, for example to- answer a GDPR erasure request. Only owners can see this list.+ answer a GDPR erasure request. Only workspace admins can see this list.</Typography>{people.error && <Alert severity="error">{people.error}</Alert>}{erasures.error && <Alert severity="error">{erasures.error}</Alert>}