Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Let a resumed erasure finish deleting files and names
Petr Bauche004d9c9a9d3
diff --git a/core/switch_core/db/models.py b/core/switch_core/db/models.pyindex da234f549..e850b2aea 100644--- a/core/switch_core/db/models.py+++ b/core/switch_core/db/models.py@@ -4183,6 +4183,12 @@ class PersonErasure(TenantScoped, Base):member_client_ids: Mapped[list] = mapped_column(JSONB, nullable=False, server_default=text("'[]'::jsonb"))+ # Files carried by messages already deleted, kept until the request's last+ # step deletes those nothing refers to, so a request resumed after a+ # restart still deletes the files of the batches before it.+ pending_file_uris: Mapped[list] = mapped_column(+ JSONB, nullable=False, server_default=text("'[]'::jsonb")+ )state: Mapped[str] = mapped_column(Text, nullable=False, default="queued")requested_by_user_id: Mapped[str | None] = mapped_column(Text, ForeignKey("users.id", ondelete="SET NULL"), nullable=Truediff --git a/core/switch_core/db/stores/erasure_store.py b/core/switch_core/db/stores/erasure_store.pyindex b4bbd16cf..bc9f6f97a 100644--- a/core/switch_core/db/stores/erasure_store.py+++ b/core/switch_core/db/stores/erasure_store.py@@ -30,10 +30,11 @@ from sqlalchemy import (or_,select,text,+ type_coerce,update,)from sqlalchemy import cast as cast_-from sqlalchemy.dialects.postgresql import array+from sqlalchemy.dialects.postgresql import JSONB, arrayfrom sqlalchemy.ext.asyncio import AsyncSessionfrom switch_core.chats import MEMBER_CLIENT_TYPE@@ -591,6 +592,66 @@ class ErasureStore:))+ async def remember_files(+ self, session: AsyncSession, erasure_id: str, uris: Collection[str]+ ) -> None:+ """Keep `uris`, the files of messages this request just deleted, on the+ request until `delete_remembered_files`, in the transaction that+ deleted the messages."""+ if not uris:+ return+ await session.execute(+ update(PersonErasure)+ .where(+ PersonErasure.tenant_id == require_tenant_id(),+ PersonErasure.id == erasure_id,+ )+ .values(+ pending_file_uris=PersonErasure.pending_file_uris.op("||")(+ type_coerce(sorted(uris), JSONB)+ )+ )+ )++ async def delete_remembered_files(+ self, session: AsyncSession, erasure_id: str+ ) -> int:+ """Delete the files the request remembered that nothing refers to now,+ and forget them all. Returns how many were deleted."""+ uris = await session.scalar(+ select(PersonErasure.pending_file_uris).where(+ PersonErasure.tenant_id == require_tenant_id(),+ PersonErasure.id == erasure_id,+ )+ )+ if not uris:+ return 0+ deleted = await self.delete_unreferenced_media(session, set(uris))+ await session.execute(+ update(PersonErasure)+ .where(+ PersonErasure.tenant_id == require_tenant_id(),+ PersonErasure.id == erasure_id,+ )+ .values(pending_file_uris=type_coerce([], JSONB))+ )+ return deleted++ async def rooms_sent_in(self, session: AsyncSession, sender_id: str) -> set[str]:+ """The bound tenant's rooms holding a message from `sender_id`."""+ return set(+ (+ await session.execute(+ select(Message.room_id)+ .where(+ Message.tenant_id == require_tenant_id(),+ Message.sender_id == sender_id,+ )+ .distinct()+ )+ ).scalars()+ )+async def delete_unreferenced_media(self, session: AsyncSession, uris: Collection[str]) -> int:diff --git a/core/switch_core/migrations/versions/e5c1b7a93d28_person_erasures.py b/core/switch_core/migrations/versions/e5c1b7a93d28_person_erasures.pyindex e4eac166b..59bb031bc 100644--- a/core/switch_core/migrations/versions/e5c1b7a93d28_person_erasures.py+++ b/core/switch_core/migrations/versions/e5c1b7a93d28_person_erasures.py@@ -70,6 +70,12 @@ def upgrade() -> None:server_default=sa.text("'[]'::jsonb"),nullable=False,),+ sa.Column(+ "pending_file_uris",+ postgresql.JSONB(astext_type=sa.Text()),+ server_default=sa.text("'[]'::jsonb"),+ nullable=False,+ ),sa.Column("state", sa.Text(), nullable=False),sa.Column("requested_by_user_id", sa.Text(), nullable=True),sa.Column(diff --git a/core/switch_core/retention/erasure.py b/core/switch_core/retention/erasure.pyindex 8e6d45511..58a83f7d1 100644--- a/core/switch_core/retention/erasure.py+++ b/core/switch_core/retention/erasure.py@@ -12,19 +12,23 @@ works the queue: for each identity it1. deletes every message the identity sent, in every room, archived onesincluded, in batches with their attachments, bridge post mappings and the- copies stored as events for agents;+ copies stored as events for agents, noting on the request the files those+ messages carried;2. stops the client that stood in for them in rooms;3. in one transaction, scrubs their name and words from approval answers andtheir name from direct rooms' names, deletes their identity row (itsclaims go by cascade), their room memberships and their client, and thenanything they sent meanwhile;4. tells the running bridge to forget them, so writing again provisions a- new identity;-5. deletes the stored files that only their messages carried.+ new identity.++Last, the request deletes the files it noted that nothing refers to any more.Every step is safe to repeat, and a request is claimed before it is worked,so one interrupted by a restart is resumed by the next pass and two processes-never work the same one. Kept: what other people wrote, including quotes and mentions of+never work the same one. Whatever a step needs from an earlier one is on the+request or still in the database, never only in memory, so a resumed request+finishes what the interrupted one started. Kept: what other people wrote, including quotes and mentions ofthem; copies on the chat platforms; their Switch account and membership,which have their own actions. `docs/design/data-retention.md` covers why."""@@ -124,6 +128,17 @@ class ErasureService:except Exception as exc:logger.exception("Erasure %s in tenant %s failed", erasure.id, tenant_id)error = f"{type(exc).__name__}: {exc}"+ # Also after a failure: the messages already deleted are gone either+ # way, and so should be the files only they carried.+ try:+ await self._delete_files(tenant_id, erasure.id)+ except Exception as exc:+ logger.exception(+ "Erasure %s in tenant %s could not delete its files",+ erasure.id,+ tenant_id,+ )+ error = error or f"{type(exc).__name__}: {exc}"async with tenant_session(self._sessions, tenant_id) as db, db.begin():finished = await self._erasures.finish(db, erasure.id, error=error)@@ -153,12 +168,10 @@ class ErasureService:return finishedasync def _erase(self, tenant_id: str, erasure_id: str, person: Person) -> None:- uris: set[str] = set()- room_ids: set[str] = set()while True:async with tenant_session(self._sessions, tenant_id) as db, db.begin():count = await self._delete_batch(- db, erasure_id, person.transport_user_id, uris, room_ids+ db, erasure_id, person.transport_user_id)if count < MESSAGE_BATCH:break@@ -175,9 +188,7 @@ class ErasureService:await self._erasures.delete_identity(db, person.external_user_id)await self._clients.delete_record(db, person.client_id)while (- await self._delete_batch(- db, erasure_id, person.transport_user_id, uris, room_ids- )+ await self._delete_batch(db, erasure_id, person.transport_user_id)== MESSAGE_BATCH):pass@@ -185,9 +196,9 @@ class ErasureService:db, erasure_id, messages=0, files=0, identities=1)- # The identity is gone from here on, so a failure below cannot be put- # right by queueing it again; it is logged rather than failing the- # request. Files missed here are taken by the hourly sweep a day later.+ # Only this process's bridges are told; one in another process, or in+ # this one after a restart, loads its people from the database when it+ # starts, and the identity is no longer there.try:await self._bridges.forget_human(person.bridge_id, person.external_user_id, person.transport_user_id@@ -200,37 +211,19 @@ class ErasureService:person.external_user_id,exc_info=True,)- try:- async with tenant_session(self._sessions, tenant_id) as db, db.begin():- files = await self._erasures.delete_unreferenced_media(db, uris)- await self._erasures.add_progress(- db, erasure_id, messages=0, files=files, identities=0- )- except Exception:- logger.warning(- "Erasure %s: could not delete the files of identity %s; the "- "hourly sweep deletes them a day later",- erasure_id,- person.external_user_id,- exc_info=True,- )async def _erase_former(self, tenant_id: str, erasure_id: str, participant: FormerParticipant) -> None:"""Erase someone whose chat app was disconnected. Their identity, clientand memberships went with the app; what is left is their messages,- their name on approval answers, and their name on direct rooms."""- uris: set[str] = set()- room_ids: set[str] = set()- while True:- async with tenant_session(self._sessions, tenant_id) as db, db.begin():- count = await self._delete_batch(- db, erasure_id, participant.sender_id, uris, room_ids- )- if count < MESSAGE_BATCH:- break+ their name on approval answers, and their name on direct rooms.++ Their messages are all that still finds them, so their names go first,+ while the rooms they wrote in can still be read off those messages: a+ request resumed after its last batch would not find them again."""async with tenant_session(self._sessions, tenant_id) as db, db.begin():+ room_ids = await self._erasures.rooms_sent_in(db, participant.sender_id)await self._erasures.scrub_approval_answers(db, participant.sender_id)await self._erasures.scrub_former_direct_rooms(db, room_ids, participant.names@@ -238,19 +231,11 @@ class ErasureService:await self._erasures.add_progress(db, erasure_id, messages=0, files=0, identities=1)- try:+ while True:async with tenant_session(self._sessions, tenant_id) as db, db.begin():- files = await self._erasures.delete_unreferenced_media(db, uris)- await self._erasures.add_progress(- db, erasure_id, messages=0, files=files, identities=0- )- except Exception:- logger.warning(- "Erasure %s: could not delete a former participant's files; the "- "hourly sweep deletes them a day later",- erasure_id,- exc_info=True,- )+ count = await self._delete_batch(db, erasure_id, participant.sender_id)+ if count < MESSAGE_BATCH:+ breakasync def _erase_member(self, tenant_id: str, erasure_id: str, member: Member@@ -258,12 +243,10 @@ class ErasureService:"""Erase what a member client sent: its messages, their files, and itsanswers to approvals. The client and its room memberships stay; aperson whose account is still in use can go on writing."""- uris: set[str] = set()- room_ids: set[str] = set()while True:async with tenant_session(self._sessions, tenant_id) as db, db.begin():count = await self._delete_batch(- db, erasure_id, member.transport_user_id, uris, room_ids+ db, erasure_id, member.transport_user_id)if count < MESSAGE_BATCH:break@@ -272,39 +255,26 @@ class ErasureService:await self._erasures.add_progress(db, erasure_id, messages=0, files=0, identities=1)- try:- async with tenant_session(self._sessions, tenant_id) as db, db.begin():- files = await self._erasures.delete_unreferenced_media(db, uris)- await self._erasures.add_progress(- db, erasure_id, messages=0, files=files, identities=0- )- except Exception:- logger.warning(- "Erasure %s: could not delete the files of member client %s; the "- "hourly sweep deletes them a day later",- erasure_id,- member.client_id,- exc_info=True,++ async def _delete_files(self, tenant_id: str, erasure_id: str) -> None:+ async with tenant_session(self._sessions, tenant_id) as db, db.begin():+ files = await self._erasures.delete_remembered_files(db, erasure_id)+ await self._erasures.add_progress(+ db, erasure_id, messages=0, files=files, identities=0)async def _delete_batch(- self,- db: AsyncSession,- erasure_id: str,- transport_user_id: str,- uris: set[str],- room_ids: set[str],+ self, db: AsyncSession, erasure_id: str, transport_user_id: str) -> int:deleted = await self._messages.delete_sent_by(db, transport_user_id=transport_user_id, limit=MESSAGE_BATCH)await self._retention.delete_bridge_mappings(db, deleted.event_ids)await self._retention.delete_agent_copies(db, deleted.message_keys)+ await self._erasures.remember_files(db, erasure_id, deleted.uris)await self._erasures.add_progress(db, erasure_id, messages=len(deleted.event_ids), files=0, identities=0)- uris |= deleted.uris- room_ids |= deleted.room_idsreturn len(deleted.event_ids)diff --git a/core/tests/switch_core/retention/test_erasure.py b/core/tests/switch_core/retention/test_erasure.pyindex 0781d0175..3194cf29d 100644--- a/core/tests/switch_core/retention/test_erasure.py+++ b/core/tests/switch_core/retention/test_erasure.py@@ -629,6 +629,99 @@ class TestErasing:assert dm.description == "a dial for erased person"+class _Killed(BaseException):+ """The process stopping mid-request: not an error the request handles."""+++async def _killed_before_its_files(+ session_factory: async_sessionmaker[AsyncSession],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Work the queued request until its last step, as a process that stops+ there would, then leave the request as a restart finds it: running, and+ stale."""++ async def killed(self: ErasureService, tenant_id: str, erasure_id: str) -> None:+ raise _Killed()++ with monkeypatch.context() as patch:+ patch.setattr(ErasureService, "_delete_files", killed)+ with pytest.raises(_Killed):+ await ErasureService(session_factory, _Clients(), _Bridges()).work_once(+ datetime.now(UTC)+ )+ async with session_factory() as session:+ await session.execute(+ update(PersonErasure).values(updated_at=NOW - timedelta(hours=1))+ )+ await session.commit()+++class TestResuming:+ async def test_a_request_stopped_after_the_identity_went_still_deletes_its_files(+ self,+ session_factory: async_sessionmaker[AsyncSession],+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ async with session_factory() as session:+ person = await _person(session, await _bridge(session), "ana")+ client = await _client_of(session, person)+ room = await _room(session, "files")+ await _blob(session, "switch-media://ana-only")+ await _say(session, room, client, "$ana", "switch-media://ana-only")+ await session.commit()+ person_id = person.id+ await _queue(session_factory, [person_id])+ await _killed_before_its_files(session_factory, monkeypatch)++ finished = await ErasureService(+ session_factory, _Clients(), _Bridges()+ ).work_once(datetime.now(UTC))++ async with session_factory() as session:+ blobs = set((await session.execute(select(MediaBlob.uri))).scalars())+ assert finished is not None and finished.state == "done"+ assert finished.files_deleted == 1+ assert finished.pending_file_uris == []+ assert blobs == set()++ async def test_a_former_participant_stopped_after_their_messages_is_still_erased(+ self,+ session_factory: async_sessionmaker[AsyncSession],+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ async with session_factory() as session:+ person = await _person(session, await _bridge(session), "ana")+ client = await _client_of(session, person)+ dm = Room(+ transport_room_id=f"!dm-{uuid.uuid4().hex[:8]}:test",+ name="Acme Slack: ana / helper",+ description="Acme Slack DM — ana / helper",+ channel_type="direct",+ )+ session.add(dm)+ await session.flush()+ await _blob(session, "switch-media://ana-only")+ await _say(session, dm, client, "$ana", "switch-media://ana-only")+ sender_id = await _disconnect(session, person)+ await session.commit()+ dm_id = dm.id+ await _queue(session_factory, [], former_sender_ids=[sender_id])+ await _killed_before_its_files(session_factory, monkeypatch)++ finished = await ErasureService(+ session_factory, _Clients(), _Bridges()+ ).work_once(datetime.now(UTC))++ async with session_factory() as session:+ blobs = set((await session.execute(select(MediaBlob.uri))).scalars())+ dm_after = await session.get(Room, dm_id)+ assert finished is not None and finished.state == "done"+ assert blobs == set()+ assert dm_after is not None+ assert dm_after.name == "Acme Slack: erased person / helper"++class TestQueue:async def test_people_are_listed_with_their_message_counts_and_claimants(self, session_factory: async_sessionmaker[AsyncSession]