Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Adapt data retention and erasure to current main

Petr Bauch93f8adc7bd5e
diff --git a/CLAUDE.md b/CLAUDE.md
index b9cf6da00..5cfde7509 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -223,8 +223,8 @@ Tests live in `core/tests/switch_core/` mirroring the module structure. Uses pyt
rotation runbook
- `docs/design/data-retention.md` — what Switch deletes: the per-workspace
message-retention window, the orphaned-file sweep, the leftover records
- pruned after a grace period, and a GDPR design note for what is not built
- (export, workspace deletion, audit-log horizon) and how erasing a person works
+ pruned after a grace period, how erasing a person works, and a GDPR design
+ note for what is not built (export, audit-log horizon)
- `docs/old/rds-migration.md` — moving a deployment's Postgres to RDS: the
proposal and the cutover runbook
diff --git a/core/switch_core/accounts/deletion.py b/core/switch_core/accounts/deletion.py
index 95f13fcd7..09be7dcb1 100644
--- a/core/switch_core/accounts/deletion.py
+++ b/core/switch_core/accounts/deletion.py
@@ -161,6 +161,7 @@ USER_REFERENCES: dict[tuple[str, str], Handling] = {
("organization_app_credentials", "updated_by"): "database clears",
("organization_service_settings", "updated_by"): "database clears",
("tenant_retention_policies", "updated_by_user_id"): "database clears",
+ ("person_erasures", "requested_by_user_id"): "database clears",
}
diff --git a/core/switch_core/accounts/workspace_deletion.py b/core/switch_core/accounts/workspace_deletion.py
index 7b2893724..52a5b88ff 100644
--- a/core/switch_core/accounts/workspace_deletion.py
+++ b/core/switch_core/accounts/workspace_deletion.py
@@ -143,6 +143,7 @@ WORKSPACE_TABLES: dict[str, Handling] = {
"tenant_data_moves": "deleted",
"tenant_join_domains": "deleted",
"tenant_retention_policies": "deleted",
+ "person_erasures": "deleted",
"tenant_usage": "deleted",
"usage_budgets": "deleted",
"workspace_service_settings": "deleted",
diff --git a/core/switch_core/db/stores/erasure_store.py b/core/switch_core/db/stores/erasure_store.py
index e16564cb7..c0bba8907 100644
--- a/core/switch_core/db/stores/erasure_store.py
+++ b/core/switch_core/db/stores/erasure_store.py
@@ -32,14 +32,13 @@ from sqlalchemy.dialects.postgresql import array
from sqlalchemy.ext.asyncio import AsyncSession
from switch_core.db.models import (
+ AgentEventRow,
ApprovalRequest,
Client,
ClientRoom,
CollaborationBridge,
ExternalUser,
ExternalUserClaim,
- HostedCutoverItem,
- HostedWakeMailbox,
Message,
PersonErasure,
Room,
@@ -270,7 +269,7 @@ class ErasureStore:
) -> PersonErasure:
"""Queue the erasure of these identities, or raise.
- Locks the tenant's erasure queue for the transaction, so two owners
+ Locks the tenant's erasure queue for the transaction, so two admins
asking at once cannot both pass the "nothing already queued" check.
"""
tenant_id = require_tenant_id()
@@ -413,24 +412,18 @@ class ErasureStore:
)
return result.scalar_one()
- async def delete_hosted_copies(
+ async def delete_agent_copies(
self, session: AsyncSession, transport_event_ids: Collection[str]
) -> None:
- """Delete the copies of these (deleted) messages held for hosted agents:
- deliveries not yet admitted, and items a cutover would import."""
+ """Delete the copies of these (deleted) messages stored as events for
+ agents (`agent_events`), which would otherwise outlive them until the
+ event retention sweep."""
if not transport_event_ids:
return
- ids = list(transport_event_ids)
await session.execute(
- delete(HostedWakeMailbox).where(
- HostedWakeMailbox.tenant_id == require_tenant_id(),
- HostedWakeMailbox.message_id.in_(ids),
- )
- )
- await session.execute(
- delete(HostedCutoverItem).where(
- HostedCutoverItem.tenant_id == require_tenant_id(),
- HostedCutoverItem.message_id.in_(ids),
+ delete(AgentEventRow).where(
+ AgentEventRow.tenant_id == require_tenant_id(),
+ AgentEventRow.message_key.in_(list(transport_event_ids)),
)
)
diff --git a/core/switch_core/db/stores/retention_store.py b/core/switch_core/db/stores/retention_store.py
index 8101096f4..3e6e61178 100644
--- a/core/switch_core/db/stores/retention_store.py
+++ b/core/switch_core/db/stores/retention_store.py
@@ -25,21 +25,17 @@ from switch_core.db.models import (
require_tenant_id,
)
-#: A stored file no attachment names and no live hosted-cutover import is
-#: waiting to carry, for a `media_blobs` row aliased `b`. Shared with erasing a
+#: A stored file no attachment names and no staged chat upload is still
+#: waiting to send, for a `media_blobs` row aliased `b`. Shared with erasing a
#: person, which deletes files by the same rule without the age condition.
UNREFERENCED_BLOB = """
NOT EXISTS (
SELECT 1 FROM message_attachments a
WHERE a.tenant_id = b.tenant_id AND a.uri = b.uri)
AND NOT EXISTS (
- SELECT 1 FROM hosted_cutover_items i
- JOIN hosted_launches l ON l.tenant_id = i.tenant_id AND l.id = i.launch_id
- AND l.state NOT IN ('deleting', 'deleted')
- CROSS JOIN LATERAL jsonb_array_elements(
- COALESCE(i.payload->'payload'->'attachments', '[]'::jsonb)) f
- WHERE i.tenant_id = b.tenant_id AND i.disposition = 'import'
- AND f->>'mxc' = b.uri)
+ SELECT 1 FROM chat_operations o
+ WHERE o.tenant_id = b.tenant_id AND o.kind = 'upload'
+ AND o.result->>'uri' = b.uri)
"""
# A blob is written before the message that carries it, in a separate
diff --git a/core/switch_core/gateway/tenants.py b/core/switch_core/gateway/tenants.py
index ad309a510..9f84ce84c 100644
--- a/core/switch_core/gateway/tenants.py
+++ b/core/switch_core/gateway/tenants.py
@@ -927,6 +927,7 @@ def _erasure_detail(erasure: PersonErasure) -> ErasureDetail:
completed_at=erasure.completed_at,
)
+
async def _require_admin_membership(
session: AsyncSession, user_store: UserStore, tenant_id: str, user: User
) -> None:
@@ -939,7 +940,6 @@ async def _require_admin_membership(
)
-
@router.get("/tenants/{tenant_id}/people")
async def list_people(
tenant_id: str,
diff --git a/core/switch_core/main.py b/core/switch_core/main.py
index f60807b44..079a5429e 100644
--- a/core/switch_core/main.py
+++ b/core/switch_core/main.py
@@ -210,10 +210,9 @@ from switch_core.process_identity import new_process_id
from switch_core.process_lease import ProcessLease
from switch_core.provisioning import Provisioning
from switch_core.provisioning.postgres import PostgresProvisioning
-from switch_core.room_changes import COALESCE_SECONDS, RoomChanges
-from switch_core.retention.service import retention_loop
from switch_core.retention.erasure import ErasureService, erasure_loop
from switch_core.retention.service import retention_loop
+from switch_core.room_changes import COALESCE_SECONDS, RoomChanges
from switch_core.room_service import RoomService
from switch_core.session_activity.listener import AgentSessionActivityListener
from switch_core.session_activity.maintenance import (
diff --git a/core/switch_core/retention/erasure.py b/core/switch_core/retention/erasure.py
index b307252d1..4917b8cb0 100644
--- a/core/switch_core/retention/erasure.py
+++ b/core/switch_core/retention/erasure.py
@@ -1,6 +1,6 @@
"""Erasing a person from a workspace.
-A workspace owner queues a request naming one or more platform identities
+A workspace admin queues a request naming one or more platform identities
(`ErasureStore.queue`), and any former participants: people from a chat app
since disconnected, whose identity rows went with it. For those only their
messages, files, approval answers and direct-room names are left to erase
@@ -8,7 +8,7 @@ messages, files, approval answers and direct-room names are left to erase
1. deletes every message the identity sent, in every room, archived ones
included, in batches with their attachments, bridge post mappings and the
- copies held for hosted agents;
+ copies stored as events for agents;
2. stops the client that stood in for them in rooms;
3. in one transaction, scrubs their name and words from approval answers and
their name from direct rooms' names, deletes their identity row (its
@@ -252,7 +252,7 @@ class ErasureService:
db, transport_user_id=transport_user_id, limit=MESSAGE_BATCH
)
await self._retention.delete_bridge_mappings(db, deleted.event_ids)
- await self._erasures.delete_hosted_copies(db, deleted.event_ids)
+ await self._erasures.delete_agent_copies(db, deleted.event_ids)
await self._erasures.add_progress(
db, erasure_id, messages=len(deleted.event_ids), files=0, identities=0
)
diff --git a/core/tests/switch_core/gateway/test_tenant_api_routes.py b/core/tests/switch_core/gateway/test_tenant_api_routes.py
index 0d37f6769..a28036324 100644
--- a/core/tests/switch_core/gateway/test_tenant_api_routes.py
+++ b/core/tests/switch_core/gateway/test_tenant_api_routes.py
@@ -49,7 +49,6 @@ from switch_core.db.models import (
Message,
Organization,
OrganizationMember,
- ProviderConnection,
Room,
ServiceTokenIssuance,
SignInFlow,
@@ -2784,7 +2783,7 @@ class TestRetentionRoutes:
self, session_factory: async_sessionmaker[AsyncSession], days: int
) -> None:
_, token = await self._member(
- session_factory, f"retention-range-{days}", "owner"
+ session_factory, f"retention-range-{days}", "admin"
)
async with _client(_app(session_factory), token) as client:
@@ -2797,7 +2796,7 @@ class TestRetentionRoutes:
async def test_another_workspace_in_the_path_is_refused(
self, session_factory: async_sessionmaker[AsyncSession]
) -> None:
- _, token = await self._member(session_factory, "retention-cross", "owner")
+ _, token = await self._member(session_factory, "retention-cross", "admin")
await _make_tenant(session_factory, TENANT_B)
async with _client(_app(session_factory), token) as client:
@@ -2888,11 +2887,11 @@ class TestErasureRoutes:
await session.commit()
return person.id
- async def test_an_owner_lists_people_queues_an_erasure_and_sees_it(
+ async def test_an_admin_lists_people_queues_an_erasure_and_sees_it(
self, session_factory: async_sessionmaker[AsyncSession]
) -> None:
await _make_tenant(session_factory, TENANT_A)
- token = await self._member(session_factory, "erase-owner", "owner")
+ token = await self._member(session_factory, "erase-admin", "admin")
person_id = await self._person(session_factory, "ana")
async with _client(_app(session_factory), token) as client:
@@ -2935,7 +2934,7 @@ class TestErasureRoutes:
self, session_factory: async_sessionmaker[AsyncSession]
) -> None:
await _make_tenant(session_factory, TENANT_A)
- token = await self._member(session_factory, "erase-former", "owner")
+ token = await self._member(session_factory, "erase-former", "admin")
sender_id = f"@switch-slack-{uuid.uuid4()}-dee:test"
async with tenant_session(session_factory, TENANT_A) as session:
room = Room(
@@ -2977,29 +2976,39 @@ class TestErasureRoutes:
assert queued.status_code == 202
assert queued.json()["identities"] == 1
- async def test_an_admin_cannot_erase_or_list_people(
+ async def test_a_member_or_an_operator_without_admin_membership_cannot_erase(
self, session_factory: async_sessionmaker[AsyncSession]
) -> None:
await _make_tenant(session_factory, TENANT_A)
- token = await self._member(session_factory, "erase-admin", "admin")
+ member = await self._member(session_factory, "erase-member", "member")
+ operator_id = await _make_member(
+ session_factory,
+ name="erase-operator",
+ tenant_id=TENANT_A,
+ role="member",
+ user_role="admin",
+ )
+ operator = create_jwt(
+ operator_id, "[email protected]", "admin", _KEYRING, TENANT_A
+ )
person_id = await self._person(session_factory, "ana")
- async with _client(_app(session_factory), token) as client:
- people = await client.get(f"/tenants/{TENANT_A}/people")
- queued = await client.post(
- f"/tenants/{TENANT_A}/erasures",
- json={"external_user_ids": [person_id]},
- )
-
- assert people.status_code == 403
- assert queued.status_code == 403
+ for token in (member, operator):
+ async with _client(_app(session_factory), token) as client:
+ people = await client.get(f"/tenants/{TENANT_A}/people")
+ queued = await client.post(
+ f"/tenants/{TENANT_A}/erasures",
+ json={"external_user_ids": [person_id]},
+ )
+ assert people.status_code == 403
+ assert queued.status_code == 403
async def test_a_person_from_another_workspace_is_not_found(
self, session_factory: async_sessionmaker[AsyncSession]
) -> None:
await _make_tenant(session_factory, TENANT_A)
await _make_tenant(session_factory, TENANT_B)
- token = await self._member(session_factory, "erase-cross", "owner")
+ token = await self._member(session_factory, "erase-cross", "admin")
async with tenant_session(session_factory, TENANT_B) as session:
other_client = Client(
transport_user_id=f"@b-{uuid.uuid4().hex[:8]}:test",
diff --git a/core/tests/switch_core/retention/test_erasure.py b/core/tests/switch_core/retention/test_erasure.py
index d04a21b98..0f965b11e 100644
--- a/core/tests/switch_core/retention/test_erasure.py
+++ b/core/tests/switch_core/retention/test_erasure.py
@@ -17,6 +17,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from switch_core.db.models import (
Agent,
+ AgentEventRow,
ApiKey,
ApprovalRequest,
AuditEvent,
@@ -25,8 +26,6 @@ from switch_core.db.models import (
CollaborationBridge,
ExternalUser,
ExternalUserClaim,
- HostedLaunch,
- HostedWakeMailbox,
MediaBlob,
Message,
MessageAttachment,
@@ -489,7 +488,7 @@ class TestErasing:
assert row.answered_by == ERASED_ANSWERER
assert row.answers == [{"question_id": "q1", "selected_option_ids": ["yes"]}]
- async def test_their_name_leaves_direct_rooms_and_hosted_copies_go(
+ async def test_their_name_leaves_direct_rooms_and_agent_copies_go(
self, session_factory: async_sessionmaker[AsyncSession]
) -> None:
async with session_factory() as session:
@@ -515,30 +514,18 @@ class TestErasing:
ClientRoom(client_id=client.id, room_id=channel.id),
]
)
- message = await _say(session, dm, client, "$to-hosted")
- owner = User(name="launch-owner", email="[email protected]", role="user")
- session.add(owner)
- await session.flush()
- launch = HostedLaunch(
- id=f"launch-{uuid.uuid4().hex[:8]}",
- owner_id=owner.id,
- name="helper",
- spec={},
- state="ready",
- agent_id=str(uuid.uuid4()),
- )
- session.add(launch)
- await session.flush()
+ message = await _say(session, dm, client, "$to-agent")
+ agent = await _agent(session)
session.add(
- HostedWakeMailbox(
- agent_id=launch.agent_id,
+ AgentEventRow(
+ agent_id=agent.id,
+ seq=1,
room_id=dm.id,
- message_id=message.transport_event_id,
- launch_id=launch.id,
- event={"body": "hello"},
- addressed_at=datetime.now(UTC),
- updated_at=datetime.now(UTC),
- expires_at=datetime.now(UTC) + timedelta(hours=1),
+ room_seq=message.seq,
+ message_key=message.transport_event_id,
+ type="message",
+ payload={"body": "hello"},
+ notifiable=True,
)
)
await session.commit()
@@ -552,14 +539,14 @@ class TestErasing:
async with session_factory() as session:
dm_after = await session.get(Room, dm_id)
channel_after = await session.get(Room, channel_id)
- mailbox = await session.scalar(
- select(func.count()).select_from(HostedWakeMailbox)
+ copies = await session.scalar(
+ select(func.count()).select_from(AgentEventRow)
)
assert dm_after is not None and channel_after is not None
assert dm_after.name == "Acme Slack: erased person / helper"
assert dm_after.description == "Acme Slack DM — erased person / helper"
assert channel_after.name == "Acme Slack: ana-fans"
- assert mailbox == 0
+ assert copies == 0
class TestQueue:
diff --git a/docs/design/data-retention.md b/docs/design/data-retention.md
index 5bca26cc9..46eb9abd7 100644
--- a/docs/design/data-retention.md
+++ b/docs/design/data-retention.md
@@ -9,7 +9,7 @@ shared picture and not from scratch.
### A message-retention window per workspace
-Each workspace keeps room messages forever until an owner or admin sets a
+Each workspace keeps room messages forever until a workspace admin sets a
window on the Workspace page ("Data retention"). With a window of N days, room
messages older than N days are deleted from every room in the workspace,
archived rooms included. Their attachments go with them, as does the record of
@@ -21,7 +21,7 @@ which chat-platform post carried each message.
- **API.** `GET`, `PUT` and `DELETE /tenants/{id}/retention` read, set and clear
the window. `GET /tenants/{id}/retention/preview?days=N` counts what a window
would delete now, which the page shows before anything is saved. All four are
- limited to owners and admins.
+ limited to workspace admins.
- **Audit.** Setting and clearing the window are recorded as
`retention_policy.set` and `retention_policy.cleared`, with the previous and
new values.
@@ -75,8 +75,8 @@ covers files left behind by retention and also those left behind when a room is
deleted, which nothing cleaned up before.
A blob is only a candidate once it is a day old. A file is uploaded before the
-message that carries it, in a separate transaction. A blob named by a pending
-hosted-cutover import is never a candidate.
+message that carries it, in a separate transaction. A blob named by a chat
+upload that is staged but not yet sent is never a candidate.
### Leftover operational records
@@ -121,9 +121,11 @@ below, in the order it is likely to be needed.
### Erase a person (built)
-A workspace owner can erase one person from the People list on the Workspace
-page, for example to answer an Art. 17 request. Admins cannot; the action
-cannot be undone.
+A workspace admin can erase one person from the People list on the Workspace
+page, for example to answer an Art. 17 request. Because it cannot be undone,
+it takes the caller's own admin membership, as deleting the workspace does:
+the deployment operator's bypass and an organization admin acting as one are
+refused.
**Who a person is.** Everyone in a room is a chat-platform identity: an
`external_users` row on one bridge, standing behind a `clients` row that is
@@ -135,7 +137,7 @@ shared claimant is a hint, not proof of a shared person.
**The flow.**
-1. The owner picks the person and confirms by typing their name. The typed
+1. The admin picks the person and confirms by typing their name. The typed
name is not stored.
2. `POST /tenants/{id}/erasures` queues a `person_erasures` row naming the
identities, and records `person_erasure.requested` (who asked, the internal
@@ -145,9 +147,8 @@ shared claimant is a hint, not proof of a shared person.
marking it running in the same statement and skipping one another process
holds, and works it. For each identity it:
- deletes every message it sent, in every room including archived ones, in
- batches with their attachments, bridge post mappings and the copies held
- for hosted agents (wake-mailbox deliveries not yet admitted, cutover
- items), keeping each room's `seq_floor` and adding to the request's
+ batches with their attachments, bridge post mappings and the copies stored
+ as events for agents (`agent_events`), keeping each room's `seq_floor` and adding to the request's
counts in the same transaction;
- stops the client that stood in for the person;
- in one transaction: drops their name and the words they typed from
@@ -161,14 +162,14 @@ shared claimant is a hint, not proof of a shared person.
- tells the running bridge to forget the identity, so writing again
provisions a new one;
- deletes the stored files that only their messages carried, with the same
- hosted-cutover guard as the hourly sweep but no grace period, because
+ staged-upload guard as the hourly sweep but no grace period, because
these files are known to have been attached.
4. The page polls `GET /tenants/{id}/erasures` for progress, and re-reads the
people list once the last request finishes. On success the request records
`person_erasure.completed` with its counts.
5. A failure before an identity's rows are deleted marks the request `failed`
with the error. It is not retried, and since the identity is still there
- the owner can queue it again. A failure after (the bridge could not be
+ the admin can queue it again. A failure after (the bridge could not be
told, the files could not be deleted) is logged as a warning and does not
fail the request: the identity is gone, so queueing again could not help.
The bridge forgets on its next restart, and the hourly sweep takes the
@@ -208,28 +209,28 @@ has the shape a bridge gives a person (`@switch-{platform}-{bridge id}-…`) and
for whom no client row is left. The People list shows them under "Disconnected
app", named by the display names their messages carry, and an erasure request
names them by that sender id (`former_sender_ids`). Erasing one deletes their
-messages, files, bridge post mappings and hosted copies, scrubs their approval
+messages, files, bridge post mappings and agent event copies, scrubs their approval
answers, and replaces their names in the direct rooms among the rooms their
messages were deleted from, since their memberships went with their client.
There is no identity left to delete and no bridge to tell. Listing them scans
-the tenant's messages, which is fine for an owner-only page opened rarely; an
+the tenant's messages, which is fine for an admin-only page opened rarely; an
index on the sender's shape would be the fix if it is not.
### Data export (not built)
-An owner or admin export of everything held about one person in the workspace,
+A workspace admin's export of everything held about one person in the workspace,
as machine-readable JSON: account fields, platform identities, and messages
they sent, with attachments. The same identity resolution as erasure applies.
This is the natural next step, since it needs the same identity lookup as
erasure (`ErasureStore.people_with_ids`).
-### Deleting a workspace (not built)
+### Deleting a workspace (built separately)
-`tenants` has no `deleted_at` today. Deleting a workspace needs an ordered
-teardown across every tenant-scoped table, because the tenant foreign keys do
-not cascade. It also needs a grace period and an audit record kept outside the
-tenant. This is a multi-tenancy-phase concern, noted here because it is the
-largest remaining erasure path.
+A workspace admin can delete a workspace once it has been emptied
+(`accounts/workspace_deletion.py`): its own setup goes with it in one
+transaction, including its retention policy and erasure requests, and the
+deletion is recorded in the server's audit log. The workspace's own audit
+history stays, which the audit-log horizon below would bound.
### The audit log
@@ -257,4 +258,4 @@ personal-data processing.
removes the rows for the messages it deletes, but not these.
- Expired controller enrolment codes. Each is tied to an `api_keys` row that
needs to go with it.
-- `tasks`, `hosted_operations` and agent-controller operation history.
+- `tasks` and agent-controller operation history.
diff --git a/gateway/src/pages/workspace/EraseDialog.tsx b/gateway/src/pages/workspace/EraseDialog.tsx
index 29a25480c..3089cab76 100644
--- a/gateway/src/pages/workspace/EraseDialog.tsx
+++ b/gateway/src/pages/workspace/EraseDialog.tsx
@@ -53,7 +53,7 @@ export default function EraseDialog({ tenantId, person, people, onClose, onQueue
// Reset only when a different person is chosen. The list behind `people` is
// refreshed while other erasures run, and resetting on every refresh would
- // undo what the owner ticked or typed just before they confirm.
+ // undo what the admin ticked or typed just before they confirm.
const personId = person?.id;
useEffect(() => {
setIncluded(new Set());
diff --git a/gateway/src/pages/workspace/PeopleSection.tsx b/gateway/src/pages/workspace/PeopleSection.tsx
index 52b11a095..0d9acf850 100644
--- a/gateway/src/pages/workspace/PeopleSection.tsx
+++ b/gateway/src/pages/workspace/PeopleSection.tsx
@@ -122,7 +122,7 @@ export default function PeopleSection({ tenantId }: { tenantId: string }) {
Everyone seen in this workspace's rooms through a chat app, including apps since
disconnected. Erasing
someone permanently deletes every message they sent and their files, for example to
- answer a GDPR erasure request. Only owners can see this list.
+ answer a GDPR erasure request. Only workspace admins can see this list.
</Typography>
{people.error && <Alert severity="error">{people.error}</Alert>}
{erasures.error && <Alert severity="error">{erasures.error}</Alert>}