Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Erase former participants of disconnected chat apps
Petr Bauch19afd80bfe29
diff --git a/core/switch_core/db/models.py b/core/switch_core/db/models.pyindex 9db79f8ab..153a7f1fe 100644--- a/core/switch_core/db/models.py+++ b/core/switch_core/db/models.py@@ -4113,7 +4113,9 @@ class PersonErasure(TenantScoped, Base):"""A workspace owner's request to erase one person from the workspace.`external_user_ids` names the platform identities to erase: one row of- `external_users` per platform the person was seen on. Erasing them deletes+ `external_users` per platform the person was seen on. `former_sender_ids`+ names people from a chat app that has since been disconnected, by the+ transport id their messages still carry. Erasing them deletesevery message they sent, the files those carried, their identity recordsand the client standing in for each, and their name on approval answers(`retention/erasure.py`). Queued here and worked by a background loop, so@@ -4134,6 +4136,11 @@ class PersonErasure(TenantScoped, Base):id: Mapped[str] = mapped_column(Text, primary_key=True, default=_uuid)external_user_ids: Mapped[list] = mapped_column(JSONB, nullable=False)+ # Former participants: people seen on a chat app since disconnected, whose+ # identity rows went with it. Named by the transport id on their messages.+ former_sender_ids: Mapped[list] = mapped_column(+ JSONB, nullable=False, server_default=text("'[]'::jsonb")+ )state: Mapped[str] = mapped_column(Text, nullable=False, default="queued")requested_by_user_id: Mapped[str | None] = mapped_column(Text, ForeignKey("users.id", ondelete="SET NULL"), nullable=Truediff --git a/core/switch_core/db/stores/erasure_store.py b/core/switch_core/db/stores/erasure_store.pyindex 4dbe4cadc..e16564cb7 100644--- a/core/switch_core/db/stores/erasure_store.py+++ b/core/switch_core/db/stores/erasure_store.py@@ -9,6 +9,7 @@ is `MessageStore.delete_sent_by`, which keeps the room's numbering intact.from __future__ import annotations+import refrom collections.abc import Collectionfrom dataclasses import dataclassfrom datetime import UTC, datetime, timedelta@@ -21,6 +22,7 @@ from sqlalchemy import (Text,delete,func,+ or_,select,text,update,@@ -58,6 +60,17 @@ ACTIVE_STATES = ("queued", "running")STALE_RUNNING = timedelta(minutes=10)+#: The transport id a bridge gives the person behind a platform account+#: (`CollaborationCore._create_human_actor`): `@switch-<platform>-<bridge id>-+#: <username>:<server>`. Agents, bridges and system clients have other shapes,+#: so this tells a person's messages apart once their identity rows are gone.+HUMAN_SENDER_PATTERN = (+ r"^@switch-([a-z]+)-"+ r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-"+)+_HUMAN_SENDER = re.compile(HUMAN_SENDER_PATTERN)++class ErasureAlreadyQueued(Exception):"""An identity named in a request already has an erasure queued or running."""@@ -87,6 +100,21 @@ class Person:claimed_by: list[Claimant]+@dataclass(frozen=True)+class FormerParticipant:+ """Someone seen on a chat app that has since been disconnected.++ Disconnecting an app deletes its identity and client rows but keeps the+ messages, so all that is left of the person is the transport id and the+ display names on what they sent.+ """++ sender_id: str+ names: list[str]+ platform: str+ message_count: int++def _rowcount(result: Result[Any]) -> int:return cast("CursorResult[Any]", result).rowcount@@ -178,11 +206,66 @@ class ErasureStore:for row in rows]+ async def list_former_participants(+ self, session: AsyncSession+ ) -> list[FormerParticipant]:+ """Everyone in the bound tenant whose messages remain but whose identity+ went with a disconnected chat app."""+ return await self._former(session, None)++ async def former_with_ids(+ self, session: AsyncSession, sender_ids: Collection[str]+ ) -> list[FormerParticipant]:+ """The named former participants that are in the bound tenant."""+ return await self._former(session, list(sender_ids))++ async def _former(+ self, session: AsyncSession, sender_ids: list[str] | None+ ) -> list[FormerParticipant]:+ tenant_id = require_tenant_id()+ has_client = (+ select(Client.id)+ .where(Client.transport_user_id == Message.sender_id)+ .correlate(Message)+ .exists()+ )+ query = (+ select(+ Message.sender_id,+ func.count(),+ func.array_agg(func.distinct(Message.sender_name)),+ )+ .where(+ Message.tenant_id == tenant_id,+ Message.sender_id.regexp_match(HUMAN_SENDER_PATTERN),+ ~has_client,+ )+ .group_by(Message.sender_id)+ .order_by(Message.sender_id)+ )+ if sender_ids is not None:+ query = query.where(Message.sender_id.in_(sender_ids))+ rows = (await session.execute(query)).all()+ former = []+ for sender_id, count, names in rows:+ match = _HUMAN_SENDER.match(sender_id)+ assert match is not None+ former.append(+ FormerParticipant(+ sender_id=sender_id,+ names=sorted(n for n in names if n),+ platform=match.group(1),+ message_count=int(count),+ )+ )+ return former+async def queue(self,session: AsyncSession,*,external_user_ids: list[str],+ former_sender_ids: list[str],requested_by_user_id: str,) -> PersonErasure:"""Queue the erasure of these identities, or raise.@@ -205,24 +288,35 @@ class ErasureStore:)).scalars())- unknown = set(external_user_ids) - known+ if former_sender_ids:+ known |= {+ f.sender_id+ for f in await self.former_with_ids(session, former_sender_ids)+ }+ unknown = (set(external_user_ids) | set(former_sender_ids)) - knownif unknown:raise UnknownIdentity(", ".join(sorted(unknown)))+ overlaps = []+ for column, ids in (+ (PersonErasure.external_user_ids, external_user_ids),+ (PersonErasure.former_sender_ids, former_sender_ids),+ ):+ if ids:+ overlaps.append(column.op("?|")(cast_(array(ids), ARRAY(Text))))overlapping = await session.scalar(select(func.count()).select_from(PersonErasure).where(PersonErasure.tenant_id == tenant_id,PersonErasure.state.in_(ACTIVE_STATES),- PersonErasure.external_user_ids.op("?|")(- cast_(array(external_user_ids), ARRAY(Text))- ),+ or_(*overlaps),))if overlapping:raise ErasureAlreadyQueued()erasure = PersonErasure(external_user_ids=list(external_user_ids),+ former_sender_ids=list(former_sender_ids),state="queued",requested_by_user_id=requested_by_user_id,)@@ -368,25 +462,47 @@ class ErasureStore:A direct room is named after the person and the agent when the bridgeadopts it. Must run while their room memberships still stand."""+ member_of = select(ClientRoom.room_id).where(+ ClientRoom.tenant_id == require_tenant_id(),+ ClientRoom.client_id == person.client_id,+ )+ return await self._scrub_direct_rooms(+ session, Room.id.in_(member_of), [person.username]+ )++ async def scrub_former_direct_rooms(+ self, session: AsyncSession, room_ids: Collection[str], names: Collection[str]+ ) -> int:+ """Take a former participant's names out of the direct rooms among+ `room_ids`, the rooms their messages were deleted from. Their room+ memberships went with their client, so these are what is left to say+ which rooms were theirs."""+ if not room_ids or not names:+ return 0+ return await self._scrub_direct_rooms(+ session, Room.id.in_(list(room_ids)), list(names)+ )++ async def _scrub_direct_rooms(+ self, session: AsyncSession, which: Any, names: list[str]+ ) -> int:rooms = (await session.execute(- select(Room)- .join(- ClientRoom,- (ClientRoom.tenant_id == Room.tenant_id)- & (ClientRoom.room_id == Room.id),- )- .where(+ select(Room).where(Room.tenant_id == require_tenant_id(),Room.channel_type == "direct",- ClientRoom.client_id == person.client_id,+ which,))).scalars()scrubbed = 0+ # Longest first, so a name that contains another is replaced whole.+ ordered = sorted(names, key=len, reverse=True)for room in rooms:- name = room.name.replace(person.username, ERASED_NAME)- description = room.description.replace(person.username, ERASED_NAME)+ name, description = room.name, room.description+ for person_name in ordered:+ name = name.replace(person_name, ERASED_NAME)+ description = description.replace(person_name, ERASED_NAME)if (name, description) != (room.name, room.description):room.name, room.description = name, descriptionscrubbed += 1diff --git a/core/switch_core/db/stores/message_store.py b/core/switch_core/db/stores/message_store.pyindex c8801a243..b6ccb7857 100644--- a/core/switch_core/db/stores/message_store.py+++ b/core/switch_core/db/stores/message_store.py@@ -25,6 +25,8 @@ class DeletedMessages:event_ids: list[str]#: The files the deleted messages carried. Others may still quote them.uris: set[str]+ #: The rooms they were deleted from.+ room_ids: set[str]class MessageStore:@@ -268,7 +270,7 @@ class MessageStore:)).all()if not rows:- return DeletedMessages(event_ids=[], uris=set())+ return DeletedMessages(event_ids=[], uris=set(), room_ids=set())message_ids = [r.id for r in rows]uris = set((@@ -291,7 +293,9 @@ class MessageStore:)await session.execute(delete(Message).where(Message.id.in_(message_ids)))return DeletedMessages(- event_ids=[row.transport_event_id for row in rows], uris=uris+ event_ids=[row.transport_event_id for row in rows],+ uris=uris,+ room_ids=set(floors),)async def head_seqs(diff --git a/core/switch_core/gateway/schemas.py b/core/switch_core/gateway/schemas.pyindex 181716e05..44559471c 100644--- a/core/switch_core/gateway/schemas.py+++ b/core/switch_core/gateway/schemas.py@@ -2057,18 +2057,32 @@ class ClaimantDetail(BaseModel):class PersonDetail(BaseModel):- """One chat-platform identity seen in the workspace's rooms."""+ """Someone seen in the workspace's rooms through a chat app.- external_user_id: str+ `kind` is `identity` for a person on a connected app, where `id` is their+ platform identity, or `former` for a person on an app since disconnected,+ where `id` is the transport id their messages carry and `bridge_name` is+ null. Erase by passing `id` in the matching list of the request.+ """++ id: str+ kind: Literal["identity", "former"]username: strplatform: str- bridge_name: str+ bridge_name: str | Nonemessage_count: intclaimed_by: list[ClaimantDetail]class ErasureCreateRequest(BaseModel):- external_user_ids: list[str] = Field(min_length=1, max_length=50)+ external_user_ids: list[str] = Field(default_factory=list, max_length=50)+ former_sender_ids: list[str] = Field(default_factory=list, max_length=50)++ @model_validator(mode="after")+ def _names_someone(self) -> ErasureCreateRequest:+ if not self.external_user_ids and not self.former_sender_ids:+ raise ValueError("name at least one person to erase")+ return selfclass ErasureDetail(BaseModel):diff --git a/core/switch_core/gateway/tenants.py b/core/switch_core/gateway/tenants.pyindex 63c3a7b0f..ad309a510 100644--- a/core/switch_core/gateway/tenants.py+++ b/core/switch_core/gateway/tenants.py@@ -917,7 +917,7 @@ def _erasure_detail(erasure: PersonErasure) -> ErasureDetail:return ErasureDetail(id=erasure.id,state=erasure.state, # type: ignore[arg-type]- identities=len(erasure.external_user_ids),+ identities=len(erasure.external_user_ids) + len(erasure.former_sender_ids),identities_erased=erasure.identities_erased,messages_deleted=erasure.messages_deleted,files_deleted=erasure.files_deleted,@@ -948,14 +948,16 @@ async def list_people(user: Annotated[User, Depends(require_tenant_admin)],user_store: Annotated[UserStore, Depends(get_user_store)],) -> list[PersonDetail]:- """Every chat-platform identity seen in the bound tenant's rooms, with how- many messages each sent and which members claim it. Admins only, by their- own admin membership: this is the list a person is erased from."""+ """Everyone seen in the bound tenant's rooms through a chat app, with how+ many messages each sent and which members claim them, including former+ participants from apps since disconnected. Admins only, by their own admin+ membership: this is the list a person is erased from."""_require_bound_tenant(tenant_id)await _require_admin_membership(session, user_store, tenant_id, user)- return [+ people = [PersonDetail(- external_user_id=person.external_user_id,+ id=person.external_user_id,+ kind="identity",username=person.username,platform=person.platform,bridge_name=person.bridge_name,@@ -967,6 +969,19 @@ async def list_people()for person in await erasure_store.list_people(session)]+ people += [+ PersonDetail(+ id=former.sender_id,+ kind="former",+ username=", ".join(former.names) or former.sender_id,+ platform=former.platform,+ bridge_name=None,+ message_count=former.message_count,+ claimed_by=[],+ )+ for former in await erasure_store.list_former_participants(session)+ ]+ return sorted(people, key=lambda p: (p.username.lower(), p.id))@router.get("/tenants/{tenant_id}/erasures")@@ -1006,9 +1021,13 @@ async def create_erasure(_require_bound_tenant(tenant_id)await _require_admin_membership(session, user_store, tenant_id, user)ids = list(dict.fromkeys(body.external_user_ids))+ former_ids = list(dict.fromkeys(body.former_sender_ids))try:erasure = await erasure_store.queue(- session, external_user_ids=ids, requested_by_user_id=user.id+ session,+ external_user_ids=ids,+ former_sender_ids=former_ids,+ requested_by_user_id=user.id,)except UnknownIdentity as exc:raise HTTPException(@@ -1025,7 +1044,7 @@ async def create_erasure(action=AuditAction.PERSON_ERASURE_REQUESTED,target_type="person_erasure",target_id=erasure.id,- details={"external_user_ids": ids},+ details={"external_user_ids": ids, "former_participants": len(former_ids)},)await session.commit()return _erasure_detail(erasure)diff --git a/core/switch_core/migrations/versions/e5c1b7a93d28_person_erasures.py b/core/switch_core/migrations/versions/e5c1b7a93d28_person_erasures.pyindex 9efb0731d..112b0fec5 100644--- a/core/switch_core/migrations/versions/e5c1b7a93d28_person_erasures.py+++ b/core/switch_core/migrations/versions/e5c1b7a93d28_person_erasures.py@@ -58,6 +58,12 @@ def upgrade() -> None:sa.Column("external_user_ids", postgresql.JSONB(astext_type=sa.Text()), nullable=False),+ sa.Column(+ "former_sender_ids",+ postgresql.JSONB(astext_type=sa.Text()),+ server_default=sa.text("'[]'::jsonb"),+ nullable=False,+ ),sa.Column("state", sa.Text(), nullable=False),sa.Column("requested_by_user_id", sa.Text(), nullable=True),sa.Column(diff --git a/core/switch_core/retention/erasure.py b/core/switch_core/retention/erasure.pyindex e1cf11443..b307252d1 100644--- a/core/switch_core/retention/erasure.py+++ b/core/switch_core/retention/erasure.py@@ -1,7 +1,10 @@"""Erasing a person from a workspace.A workspace owner queues a request naming one or more platform identities-(`ErasureStore.queue`). `erasure_loop` works the queue: for each identity it+(`ErasureStore.queue`), and any former participants: people from a chat app+since disconnected, whose identity rows went with it. For those only their+messages, files, approval answers and direct-room names are left to erase+(`_erase_former`). `erasure_loop` works the queue: for each identity it1. deletes every message the identity sent, in every room, archived onesincluded, in batches with their attachments, bridge post mappings and the@@ -34,7 +37,11 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmakerfrom switch_core.db.audit import AuditAction, record_audit_eventfrom switch_core.db.models import PersonErasure, require_tenant_idfrom switch_core.db.session_scope import tenant_session-from switch_core.db.stores.erasure_store import ErasureStore, Person+from switch_core.db.stores.erasure_store import (+ ErasureStore,+ FormerParticipant,+ Person,+)from switch_core.db.stores.message_store import MessageStorefrom switch_core.db.stores.retention_store import RetentionStorefrom switch_core.db.tenant_lookup import all_tenant_ids@@ -95,6 +102,13 @@ class ErasureService:)for person in people:await self._erase(tenant_id, erasure.id, person)+ if erasure.former_sender_ids:+ async with tenant_session(self._sessions, tenant_id) as db:+ former = await self._erasures.former_with_ids(+ db, erasure.former_sender_ids+ )+ for participant in former:+ await self._erase_former(tenant_id, erasure.id, participant)except Exception as exc:logger.exception("Erasure %s in tenant %s failed", erasure.id, tenant_id)error = f"{type(exc).__name__}: {exc}"@@ -128,9 +142,12 @@ class ErasureService:async def _erase(self, tenant_id: str, erasure_id: str, person: Person) -> None:uris: set[str] = set()+ room_ids: set[str] = set()while True:async with tenant_session(self._sessions, tenant_id) as db, db.begin():- count = await self._delete_batch(db, erasure_id, person, uris)+ count = await self._delete_batch(+ db, erasure_id, person.transport_user_id, uris, room_ids+ )if count < MESSAGE_BATCH:break@@ -146,7 +163,10 @@ class ErasureService:await self._erasures.delete_identity(db, person.external_user_id)await self._clients.delete_record(db, person.client_id)while (- await self._delete_batch(db, erasure_id, person, uris) == MESSAGE_BATCH+ await self._delete_batch(+ db, erasure_id, person.transport_user_id, uris, room_ids+ )+ == MESSAGE_BATCH):passawait self._erasures.add_progress(@@ -183,11 +203,53 @@ class ErasureService:exc_info=True,)+ async def _erase_former(+ self, tenant_id: str, erasure_id: str, participant: FormerParticipant+ ) -> None:+ """Erase someone whose chat app was disconnected. Their identity, client+ and memberships went with the app; what is left is their messages,+ their name on approval answers, and their name on direct rooms."""+ uris: set[str] = set()+ room_ids: set[str] = set()+ while True:+ async with tenant_session(self._sessions, tenant_id) as db, db.begin():+ count = await self._delete_batch(+ db, erasure_id, participant.sender_id, uris, room_ids+ )+ if count < MESSAGE_BATCH:+ break+ async with tenant_session(self._sessions, tenant_id) as db, db.begin():+ await self._erasures.scrub_approval_answers(db, participant.sender_id)+ await self._erasures.scrub_former_direct_rooms(+ db, room_ids, participant.names+ )+ await self._erasures.add_progress(+ db, erasure_id, messages=0, files=0, identities=1+ )+ try:+ async with tenant_session(self._sessions, tenant_id) as db, db.begin():+ files = await self._erasures.delete_unreferenced_media(db, uris)+ await self._erasures.add_progress(+ db, erasure_id, messages=0, files=files, identities=0+ )+ except Exception:+ logger.warning(+ "Erasure %s: could not delete a former participant's files; the "+ "hourly sweep deletes them a day later",+ erasure_id,+ exc_info=True,+ )+async def _delete_batch(- self, db: AsyncSession, erasure_id: str, person: Person, uris: set[str]+ self,+ db: AsyncSession,+ erasure_id: str,+ transport_user_id: str,+ uris: set[str],+ room_ids: set[str],) -> int:deleted = await self._messages.delete_sent_by(- db, transport_user_id=person.transport_user_id, limit=MESSAGE_BATCH+ db, transport_user_id=transport_user_id, limit=MESSAGE_BATCH)await self._retention.delete_bridge_mappings(db, deleted.event_ids)await self._erasures.delete_hosted_copies(db, deleted.event_ids)@@ -195,6 +257,7 @@ class ErasureService:db, erasure_id, messages=len(deleted.event_ids), files=0, identities=0)uris |= deleted.uris+ room_ids |= deleted.room_idsreturn len(deleted.event_ids)diff --git a/core/tests/switch_core/gateway/test_tenant_api_routes.py b/core/tests/switch_core/gateway/test_tenant_api_routes.pyindex 99638eced..0d37f6769 100644--- a/core/tests/switch_core/gateway/test_tenant_api_routes.py+++ b/core/tests/switch_core/gateway/test_tenant_api_routes.py@@ -2925,9 +2925,58 @@ class TestErasureRoutes:)).all()assert actions == [- ("person_erasure.requested", {"external_user_ids": [person_id]})+ (+ "person_erasure.requested",+ {"external_user_ids": [person_id], "former_participants": 0},+ )]+ async def test_a_former_participant_is_listed_and_erased(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ await _make_tenant(session_factory, TENANT_A)+ token = await self._member(session_factory, "erase-former", "owner")+ sender_id = f"@switch-slack-{uuid.uuid4()}-dee:test"+ async with tenant_session(session_factory, TENANT_A) as session:+ room = Room(+ transport_room_id=f"!r-{uuid.uuid4().hex[:8]}:test",+ name="r",+ description="r",+ )+ session.add(room)+ await session.flush()+ await MessageStore().create(+ session,+ Message(+ room_id=room.id,+ transport_event_id="$dee",+ sender_id=sender_id,+ sender_name="dee",+ event_type="m.room.message",+ msgtype="m.text",+ body="hi",+ content={"msgtype": "m.text", "body": "hi"},+ ),+ [],+ )+ await session.commit()++ async with _client(_app(session_factory), token) as client:+ people = await client.get(f"/tenants/{TENANT_A}/people")+ empty = await client.post(f"/tenants/{TENANT_A}/erasures", json={})+ queued = await client.post(+ f"/tenants/{TENANT_A}/erasures",+ json={"former_sender_ids": [sender_id]},+ )++ assert [+ (p["id"], p["kind"], p["username"], p["bridge_name"], p["message_count"])+ for p in people.json()+ ] == [(sender_id, "former", "dee", None, 1)]+ assert empty.status_code == 422+ assert queued.status_code == 202+ assert queued.json()["identities"] == 1+async def test_an_admin_cannot_erase_or_list_people(self, session_factory: async_sessionmaker[AsyncSession]) -> None:diff --git a/core/tests/switch_core/retention/test_erasure.py b/core/tests/switch_core/retention/test_erasure.pyindex 537283cc1..d04a21b98 100644--- a/core/tests/switch_core/retention/test_erasure.py+++ b/core/tests/switch_core/retention/test_erasure.py@@ -12,7 +12,7 @@ import uuidfrom datetime import UTC, datetime, timedeltaimport pytest-from sqlalchemy import func, select+from sqlalchemy import func, select, updatefrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmakerfrom switch_core.db.models import (@@ -191,7 +191,9 @@ async def _agent(session: AsyncSession) -> Agent:async def _queue(- session_factory: async_sessionmaker[AsyncSession], external_user_ids: list[str]+ session_factory: async_sessionmaker[AsyncSession],+ external_user_ids: list[str],+ former_sender_ids: list[str] | None = None,) -> str:async with session_factory() as session:owner = User(@@ -202,12 +204,31 @@ async def _queue(session.add(owner)await session.flush()erasure = await ErasureStore().queue(- session, external_user_ids=external_user_ids, requested_by_user_id=owner.id+ session,+ external_user_ids=external_user_ids,+ former_sender_ids=former_sender_ids or [],+ requested_by_user_id=owner.id,)await session.commit()return erasure.id+async def _disconnect(session: AsyncSession, person: ExternalUser) -> str:+ """What removing their chat app does: identity and client go, messages stay."""+ client = await _client_of(session, person)+ await session.execute(+ update(Message)+ .where(Message.sender_client_id == client.id)+ .values(sender_name=person.external_username)+ )+ sender_id = client.transport_user_id+ await session.delete(person)+ await session.flush()+ await session.delete(client)+ await session.flush()+ return sender_id++class TestErasing:async def test_everything_they_sent_goes_and_everything_else_stays(self, session_factory: async_sessionmaker[AsyncSession]@@ -587,3 +608,112 @@ class TestQueue:with pytest.raises(ErasureAlreadyQueued):await _queue(session_factory, [bo_id, ana_id])await _queue(session_factory, [bo_id])+++class TestFormerParticipants:+ async def test_people_of_a_disconnected_app_are_listed_and_no_one_else(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ bridge = await _bridge(session)+ ana = await _person(session, bridge, "ana")+ bo = await _person(session, bridge, "bo")+ agent = await _agent(session)+ agent_client = await session.get(Client, agent.client_id)+ assert agent_client is not None+ room = await _room(session, "r")+ for index in range(2):+ await _say(session, room, await _client_of(session, ana), f"$a{index}")+ await _say(session, room, await _client_of(session, bo), "$b")+ await _say(session, room, agent_client, "$agent")+ ana_sender = await _disconnect(session, ana)+ await session.commit()++ former = await ErasureStore().list_former_participants(session)++ assert [+ (f.sender_id, f.names, f.platform, f.message_count) for f in former+ ] == [(ana_sender, ["ana"], "slack", 2)]++ async def test_a_former_participant_is_erased_and_the_rest_stays(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ bridge = await _bridge(session)+ ana = await _person(session, bridge, "ana")+ bo = await _person(session, bridge, "bo")+ ana_client = await _client_of(session, ana)+ bo_client = await _client_of(session, bo)+ agent = await _agent(session)+ room = await _room(session, "r", archived=True)+ dm = Room(+ transport_room_id=f"!dm-{uuid.uuid4().hex[:8]}:test",+ name="Acme Slack: ana / helper",+ description="Acme Slack DM — ana / helper",+ channel_type="direct",+ )+ session.add(dm)+ await session.flush()+ await _blob(session, "mxc://test/ana-only")+ await _blob(session, "mxc://test/shared")+ await _say(session, room, ana_client, "$a1", "mxc://test/ana-only")+ await _say(session, dm, ana_client, "$a2", "mxc://test/shared")+ await _say(session, room, bo_client, "$b1", "mxc://test/shared")+ session.add(+ ApprovalRequest(+ agent_id=agent.id,+ session_id="s",+ request_id="q",+ turn_id="t",+ kind="questions",+ title="t",+ options=[],+ questions=[],+ state="answered",+ answered_by=ana_client.transport_user_id,+ answers=[{"question_id": "q1", "custom_text": "private"}],+ )+ )+ ana_sender = await _disconnect(session, ana)+ await session.commit()+ dm_id = dm.id+ erasure_id = await _queue(session_factory, [], [ana_sender])++ finished = await ErasureService(+ session_factory, _Clients(), _Bridges()+ ).work_once(datetime.now(UTC))++ assert finished is not None and finished.id == erasure_id+ assert (finished.state, finished.error) == ("done", None)+ assert (+ finished.messages_deleted,+ finished.files_deleted,+ finished.identities_erased,+ ) == (2, 1, 1)+ async with session_factory() as session:+ left = (await session.execute(select(Message.transport_event_id))).scalars()+ blobs = (await session.execute(select(MediaBlob.uri))).scalars()+ approval = (await session.execute(select(ApprovalRequest))).scalar_one()+ dm_after = await session.get(Room, dm_id)+ former = await ErasureStore().list_former_participants(session)+ assert list(left) == ["$b1"]+ assert list(blobs) == ["mxc://test/shared"]+ assert approval.answered_by == ERASED_ANSWERER+ assert approval.answers == [{"question_id": "q1"}]+ assert dm_after is not None+ assert dm_after.name == "Acme Slack: erased person / helper"+ assert former == []++ async def test_a_sender_that_is_not_a_former_participant_is_refused(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ ana = await _person(session, await _bridge(session), "ana")+ await _say(+ session, await _room(session, "r"), await _client_of(session, ana), "$a"+ )+ await session.commit()+ live_sender = (await _client_of(session, ana)).transport_user_id++ with pytest.raises(UnknownIdentity):+ await _queue(session_factory, [], [live_sender])diff --git a/docs/design/data-retention.md b/docs/design/data-retention.mdindex d1b1206b0..5bca26cc9 100644--- a/docs/design/data-retention.md+++ b/docs/design/data-retention.md@@ -202,9 +202,18 @@ sent deleted, nothing would show a placeholder anyway.channel's name is the platform's.**People from a disconnected chat app.** Disconnecting an app deletes its-identity and client rows but keeps their messages in the detached rooms, so-those people are not in the People list and cannot be erased this way. See-"Not covered yet".+identity and client rows but keeps their messages in the detached rooms. These+former participants are found by their messages instead: every sender whose id+has the shape a bridge gives a person (`@switch-{platform}-{bridge id}-…`) and+for whom no client row is left. The People list shows them under "Disconnected+app", named by the display names their messages carry, and an erasure request+names them by that sender id (`former_sender_ids`). Erasing one deletes their+messages, files, bridge post mappings and hosted copies, scrubs their approval+answers, and replaces their names in the direct rooms among the rooms their+messages were deleted from, since their memberships went with their client.+There is no identity left to delete and no bridge to tell. Listing them scans+the tenant's messages, which is fine for an owner-only page opened rarely; an+index on the sender's shape would be the fix if it is not.### Data export (not built)@@ -249,6 +258,3 @@ personal-data processing.- Expired controller enrolment codes. Each is tied to an `api_keys` row thatneeds to go with it.- `tasks`, `hosted_operations` and agent-controller operation history.-- Erasing someone seen only on a chat app that has since been disconnected.- Their messages carry their transport id and display name, but no identity- row is left to find them by.diff --git a/gateway/src/data/api.ts b/gateway/src/data/api.tsindex f6655ddce..88fe66289 100644--- a/gateway/src/data/api.ts+++ b/gateway/src/data/api.ts@@ -1533,11 +1533,15 @@ export interface PersonClaimant {name: string;}+/** `identity`: someone on a connected app, `id` is their platform identity.+ * `former`: someone on an app since disconnected, `id` is the sender id their+ * messages carry, and `bridge_name` is null. */export interface Person {- external_user_id: string;+ id: string;+ kind: "identity" | "former";username: string;platform: string;- bridge_name: string;+ bridge_name: string | null;message_count: number;claimed_by: PersonClaimant[];}@@ -1565,12 +1569,10 @@ export async function fetchErasures(tenantId: string): Promise<Erasure[]> {return jsonRequest<Erasure[]>(`/tenants/${encodeURIComponent(tenantId)}/erasures`, "GET");}-export async function eraseIdentities(- tenantId: string,- externalUserIds: string[],-): Promise<Erasure> {+export async function erasePeople(tenantId: string, people: Person[]): Promise<Erasure> {return jsonRequest<Erasure>(`/tenants/${encodeURIComponent(tenantId)}/erasures`, "POST", {- external_user_ids: externalUserIds,+ external_user_ids: people.filter((p) => p.kind === "identity").map((p) => p.id),+ former_sender_ids: people.filter((p) => p.kind === "former").map((p) => p.id),});}diff --git a/gateway/src/pages/workspace/EraseDialog.test.tsx b/gateway/src/pages/workspace/EraseDialog.test.tsxindex 1ff8776e2..1d96b463e 100644--- a/gateway/src/pages/workspace/EraseDialog.test.tsx+++ b/gateway/src/pages/workspace/EraseDialog.test.tsx@@ -6,7 +6,8 @@ import { describeErasure } from "./PeopleSection";function person(id: string, username: string, claimants: string[], messages = 0): Person {return {- external_user_id: id,+ id,+ kind: "identity",username,platform: "slack",bridge_name: `Bridge ${id}`,@@ -79,7 +80,7 @@ describe("EraseDialog", () => {await waitFor(() => expect(onQueued).toHaveBeenCalled());const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit];- expect(JSON.parse(String(init.body))).toEqual({ external_user_ids: ["a"] });+ expect(JSON.parse(String(init.body))).toEqual({ external_user_ids: ["a"], former_sender_ids: [] });});it("keeps what was ticked and typed when the people list refreshes", async () => {@@ -96,6 +97,39 @@ describe("EraseDialog", () => {fireEvent.click(screen.getByRole("button", { name: "Erase" }));await waitFor(() => expect(fetchMock).toHaveBeenCalled());const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit];- expect(JSON.parse(String(init.body))).toEqual({ external_user_ids: ["a", "b"] });+ expect(JSON.parse(String(init.body))).toEqual({ external_user_ids: ["a", "b"], former_sender_ids: [] });+ });++ it("erases a former participant by the sender id their messages carry", async () => {+ const fetchMock = vi.fn(async () => new Response(JSON.stringify({}), { status: 202 }));+ vi.stubGlobal("fetch", fetchMock);+ const former: Person = {+ id: "@switch-slack-x-dee:example.org",+ kind: "former",+ username: "dee",+ platform: "slack",+ bridge_name: null,+ message_count: 4,+ claimed_by: [],+ };+ render(+ <EraseDialog+ tenantId="t1"+ person={former}+ people={[former, ana]}+ onClose={() => {}}+ onQueued={() => {}}+ />,+ );++ expect(screen.getByText(/the disconnected app/)).toBeTruthy();+ fireEvent.change(screen.getByLabelText("Type dee to confirm"), { target: { value: "dee" } });+ fireEvent.click(screen.getByRole("button", { name: "Erase" }));+ await waitFor(() => expect(fetchMock).toHaveBeenCalled());+ const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit];+ expect(JSON.parse(String(init.body))).toEqual({+ external_user_ids: [],+ former_sender_ids: ["@switch-slack-x-dee:example.org"],+ });});});diff --git a/gateway/src/pages/workspace/EraseDialog.tsx b/gateway/src/pages/workspace/EraseDialog.tsxindex 44c0b72e2..29a25480c 100644--- a/gateway/src/pages/workspace/EraseDialog.tsx+++ b/gateway/src/pages/workspace/EraseDialog.tsx@@ -13,7 +13,7 @@ import {Typography,} from "@mui/material";import { useEffect, useMemo, useState } from "react";-import { type Person, eraseIdentities } from "../../data/api";+import { type Person, erasePeople } from "../../data/api";/** Identities that may be the same person as `person`: every one claimed by* a member who also claims `person`. Only a hint, offered unticked: claims are@@ -24,7 +24,7 @@ export function relatedIdentities(person: Person, people: Person[]): Person[] {if (claimants.size === 0) return [];return people.filter((other) =>- other.external_user_id !== person.external_user_id &&+ other.id !== person.id &&other.claimed_by.some((c) => claimants.has(c.user_id)),);}@@ -54,7 +54,7 @@ export default function EraseDialog({ tenantId, person, people, onClose, onQueue// Reset only when a different person is chosen. The list behind `people` is// refreshed while other erasures run, and resetting on every refresh would// undo what the owner ticked or typed just before they confirm.- const personId = person?.external_user_id;+ const personId = person?.id;useEffect(() => {setIncluded(new Set());setTyped("");@@ -63,7 +63,7 @@ export default function EraseDialog({ tenantId, person, people, onClose, onQueueif (!person) return null;- const chosen = [person, ...related.filter((p) => included.has(p.external_user_id))];+ const chosen = [person, ...related.filter((p) => included.has(p.id))];const messages = chosen.reduce((sum, p) => sum + p.message_count, 0);const confirmed = typed === person.username;@@ -79,10 +79,7 @@ export default function EraseDialog({ tenantId, person, people, onClose, onQueuesetSubmitting(true);setError(null);try {- await eraseIdentities(- tenantId,- chosen.map((p) => p.external_user_id),- );+ await erasePeople(tenantId, chosen);onQueued();onClose();} catch (err) {@@ -104,8 +101,9 @@ export default function EraseDialog({ tenantId, person, people, onClose, onQueue</DialogContentText><DialogContentText>What other people wrote, including replies that quote them, is kept. Copies in{" "}- {person.bridge_name} and other chat apps are not deleted; remove those in the app.- Their Switch account and membership, if any, are not affected.+ {person.bridge_name ?? "the disconnected app"} and other chat apps are not+ deleted; remove those in the app. Their Switch account and membership, if any, are+ not affected.</DialogContentText>{related.length > 0 && (<Stack spacing={0.5}>@@ -115,14 +113,14 @@ export default function EraseDialog({ tenantId, person, people, onClose, onQueue</Typography>{related.map((other) => (<FormControlLabel- key={other.external_user_id}+ key={other.id}control={<Checkbox- checked={included.has(other.external_user_id)}- onChange={() => toggle(other.external_user_id)}+ checked={included.has(other.id)}+ onChange={() => toggle(other.id)}/>}- label={`${other.username} on ${other.bridge_name} (${plural(other.message_count, "message")})`}+ label={`${other.username} on ${other.bridge_name ?? "a disconnected app"} (${plural(other.message_count, "message")})`}/>))}</Stack>diff --git a/gateway/src/pages/workspace/PeopleSection.tsx b/gateway/src/pages/workspace/PeopleSection.tsxindex 1709b7676..52b11a095 100644--- a/gateway/src/pages/workspace/PeopleSection.tsx+++ b/gateway/src/pages/workspace/PeopleSection.tsx@@ -15,8 +15,6 @@ import { formatDate, titleCase } from "../../theme/hootFormat";import EraseDialog from "./EraseDialog";import { useLoad } from "./useLoad";-type PersonRow = Person & { id: string };-const POLL_MS = 3000;export function describeErasure(erasure: Erasure): string {@@ -67,7 +65,7 @@ export default function PeopleSection({ tenantId }: { tenantId: string }) {wasActive.current = active;}, [active, refetchPeople]);- const rows = useMemo<PersonRow[]>(() => {+ const rows = useMemo<Person[]>(() => {const needle = search.trim().toLowerCase();return (people.data ?? []).filter(@@ -75,11 +73,10 @@ export default function PeopleSection({ tenantId }: { tenantId: string }) {!needle ||p.username.toLowerCase().includes(needle) ||p.claimed_by.some((c) => c.name.toLowerCase().includes(needle)),- )- .map((p) => ({ ...p, id: p.external_user_id }));+ );}, [people.data, search]);- const columns = useMemo<GridColDef<PersonRow>[]>(+ const columns = useMemo<GridColDef<Person>[]>(() => [{ field: "username", headerName: "Name", flex: 1, minWidth: 160 },{@@ -87,7 +84,8 @@ export default function PeopleSection({ tenantId }: { tenantId: string }) {headerName: "Seen on",flex: 1,minWidth: 160,- valueGetter: (_value, row) => `${row.bridge_name} (${titleCase(row.platform)})`,+ valueGetter: (_value, row) =>+ `${row.bridge_name ?? "Disconnected app"} (${titleCase(row.platform)})`,},{ field: "message_count", headerName: "Messages", width: 110, type: "number" },{@@ -121,7 +119,8 @@ export default function PeopleSection({ tenantId }: { tenantId: string }) {<Stack spacing={1.5}><Typography variant="h6">People</Typography><Typography variant="body2" sx={{ color: "text.secondary" }}>- Everyone seen in this workspace's rooms through a connected chat app. Erasing+ Everyone seen in this workspace's rooms through a chat app, including apps since+ disconnected. Erasingsomeone permanently deletes every message they sent and their files, for example toanswer a GDPR erasure request. Only owners can see this list.</Typography>