Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Delete agents' copies of deleted messages by message key, in retention too
Petr Bauche240b9e14e08
diff --git a/core/switch_core/db/stores/erasure_store.py b/core/switch_core/db/stores/erasure_store.pyindex 1d21c28ba..dbe7dfe1b 100644--- a/core/switch_core/db/stores/erasure_store.py+++ b/core/switch_core/db/stores/erasure_store.py@@ -38,7 +38,6 @@ from sqlalchemy.ext.asyncio import AsyncSessionfrom switch_core.chats import MEMBER_CLIENT_TYPEfrom switch_core.db.models import (- AgentEventRow,ApprovalRequest,Client,ClientRoom,@@ -496,21 +495,6 @@ class ErasureStore:)return result.scalar_one()- async def delete_agent_copies(- self, session: AsyncSession, transport_event_ids: Collection[str]- ) -> None:- """Delete the copies of these (deleted) messages stored as events for- agents (`agent_events`), which would otherwise outlive them until the- event retention sweep."""- if not transport_event_ids:- return- await session.execute(- delete(AgentEventRow).where(- AgentEventRow.tenant_id == require_tenant_id(),- AgentEventRow.message_key.in_(list(transport_event_ids)),- )- )-async def scrub_approval_answers(self, session: AsyncSession, transport_user_id: str) -> int:diff --git a/core/switch_core/db/stores/message_store.py b/core/switch_core/db/stores/message_store.pyindex b6ccb7857..4845b2e09 100644--- a/core/switch_core/db/stores/message_store.py+++ b/core/switch_core/db/stores/message_store.py@@ -14,6 +14,7 @@ from sqlalchemy import ()from sqlalchemy.ext.asyncio import AsyncSession+from switch_core.attachments import ATTACHMENT_GROUP_KEYfrom switch_core.db.id_lists import any_offrom switch_core.db.models import Message, MessageAttachment, Room, require_tenant_idfrom switch_core.db.notify_ddl import SET_WRITER_ANNOUNCES@@ -23,6 +24,9 @@ from switch_core.db.raw import RawStatement@dataclass(frozen=True)class DeletedMessages:event_ids: list[str]+ #: The keys agents' copies of them are stored under (`agent_events.message_key`):+ #: the attachment group id for a file sent as part of several, else the event id.+ message_keys: set[str]#: The files the deleted messages carried. Others may still quote them.uris: set[str]#: The rooms they were deleted from.@@ -220,16 +224,13 @@ class MessageStore:async def delete_sent_before(self, session: AsyncSession, cutoff: datetime, *, limit: int- ) -> list[str]:+ ) -> DeletedMessages:"""Delete up to `limit` of the bound tenant's oldest messages sent before- `cutoff`, with their attachments. Returns their transport event ids.+ `cutoff`, with their attachments.See `_delete_where` for what else changes and what does not."""- deleted = await self._delete_where(- session, Message.sent_at < cutoff, limit=limit- )- return deleted.event_ids+ return await self._delete_where(session, Message.sent_at < cutoff, limit=limit)async def delete_sent_by(self, session: AsyncSession, *, transport_user_id: str, limit: int@@ -262,7 +263,14 @@ class MessageStore:rows = (await session.execute(select(- Message.id, Message.room_id, Message.seq, Message.transport_event_id+ Message.id,+ Message.room_id,+ Message.seq,+ Message.transport_event_id,+ func.coalesce(+ Message.content[ATTACHMENT_GROUP_KEY]["id"].astext,+ Message.transport_event_id,+ ).label("message_key"),).where(Message.tenant_id == require_tenant_id(), condition).order_by(Message.sent_at)@@ -270,7 +278,9 @@ class MessageStore:)).all()if not rows:- return DeletedMessages(event_ids=[], uris=set(), room_ids=set())+ return DeletedMessages(+ event_ids=[], message_keys=set(), uris=set(), room_ids=set()+ )message_ids = [r.id for r in rows]uris = set((@@ -294,6 +304,7 @@ class MessageStore:await session.execute(delete(Message).where(Message.id.in_(message_ids)))return DeletedMessages(event_ids=[row.transport_event_id for row in rows],+ message_keys={row.message_key for row in rows},uris=uris,room_ids=set(floors),)diff --git a/core/switch_core/db/stores/retention_store.py b/core/switch_core/db/stores/retention_store.pyindex 3e6e61178..5d5f0b642 100644--- a/core/switch_core/db/stores/retention_store.py+++ b/core/switch_core/db/stores/retention_store.py@@ -16,6 +16,7 @@ from sqlalchemy.dialects.postgresql import insertfrom sqlalchemy.ext.asyncio import AsyncSessionfrom switch_core.db.models import (+ AgentEventRow,ApprovalRequest,BridgeMessageMap,Invitation,@@ -119,6 +120,21 @@ class RetentionStore:)return _rowcount(result)+ async def delete_agent_copies(+ self, session: AsyncSession, message_keys: Collection[str]+ ) -> None:+ """Delete the copies of these (deleted) messages stored as events for+ agents (`agent_events`), keyed as `DeletedMessages.message_keys`, which+ would otherwise outlive them until the event retention sweep."""+ if not message_keys:+ return+ await session.execute(+ delete(AgentEventRow).where(+ AgentEventRow.tenant_id == require_tenant_id(),+ AgentEventRow.message_key.in_(list(message_keys)),+ )+ )+async def delete_unreferenced_media(self, session: AsyncSession, *, created_before: datetime, limit: int) -> int:diff --git a/core/switch_core/retention/erasure.py b/core/switch_core/retention/erasure.pyindex d3e57ea18..8e6d45511 100644--- a/core/switch_core/retention/erasure.py+++ b/core/switch_core/retention/erasure.py@@ -299,7 +299,7 @@ class ErasureService:db, transport_user_id=transport_user_id, limit=MESSAGE_BATCH)await self._retention.delete_bridge_mappings(db, deleted.event_ids)- await self._erasures.delete_agent_copies(db, deleted.event_ids)+ await self._retention.delete_agent_copies(db, deleted.message_keys)await self._erasures.add_progress(db, erasure_id, messages=len(deleted.event_ids), files=0, identities=0)diff --git a/core/switch_core/retention/service.py b/core/switch_core/retention/service.pyindex 88d5b014f..3a6ba4677 100644--- a/core/switch_core/retention/service.py+++ b/core/switch_core/retention/service.py@@ -154,12 +154,13 @@ class RetentionService:deleted = 0while True:async with tenant_session(self._sessions, tenant_id) as db, db.begin():- event_ids = await self._messages.delete_sent_before(+ batch = await self._messages.delete_sent_before(db, cutoff, limit=MESSAGE_BATCH)- await self._retention.delete_bridge_mappings(db, event_ids)- deleted += len(event_ids)- if len(event_ids) < MESSAGE_BATCH:+ await self._retention.delete_bridge_mappings(db, batch.event_ids)+ await self._retention.delete_agent_copies(db, batch.message_keys)+ deleted += len(batch.event_ids)+ if len(batch.event_ids) < MESSAGE_BATCH:return deleted, Falseif time.monotonic() >= deadline:return deleted, Truediff --git a/core/tests/switch_core/retention/test_erasure.py b/core/tests/switch_core/retention/test_erasure.pyindex ccf03d252..1b98ce224 100644--- a/core/tests/switch_core/retention/test_erasure.py+++ b/core/tests/switch_core/retention/test_erasure.py@@ -15,6 +15,7 @@ import pytestfrom sqlalchemy import func, select, updatefrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker+from switch_core.attachments import ATTACHMENT_GROUP_KEYfrom switch_core.db.models import (Agent,AgentEventRow,@@ -550,6 +551,62 @@ class TestErasing:assert channel_after.name == "Acme Slack: ana-fans"assert copies == 0+ async def test_agent_copies_of_files_sent_together_go(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ person = await _person(session, await _bridge(session), "ana")+ client = await _client_of(session, person)+ room = await _room(session, "files")+ for event_id in ("$part-1", "$part-2"):+ part = await MessageStore().create(+ session,+ Message(+ room_id=room.id,+ transport_event_id=event_id,+ sender_id=client.transport_user_id,+ sender_client_id=client.id,+ event_type="m.room.message",+ msgtype="m.file",+ body=event_id,+ content={+ "msgtype": "m.file",+ "body": event_id,+ ATTACHMENT_GROUP_KEY: {"id": "group-ana", "count": 2},+ },+ ),+ [],+ )+ other = await _person(session, await _bridge(session), "bo")+ kept = await _say(session, room, await _client_of(session, other), "$bo")+ agent = await _agent(session)+ for seq, key in enumerate(("group-ana", kept.transport_event_id), start=1):+ session.add(+ AgentEventRow(+ agent_id=agent.id,+ seq=seq,+ room_id=room.id,+ room_seq=part.seq,+ message_key=key,+ type="message",+ payload={"body": key},+ notifiable=True,+ )+ )+ await session.commit()+ person_id = person.id+ await _queue(session_factory, [person_id])++ await ErasureService(session_factory, _Clients(), _Bridges()).work_once(+ datetime.now(UTC)+ )++ async with session_factory() as session:+ copies = set(+ (await session.execute(select(AgentEventRow.message_key))).scalars()+ )+ assert copies == {"$bo"}+class TestQueue:async def test_people_are_listed_with_their_message_counts_and_claimants(diff --git a/core/tests/switch_core/retention/test_service.py b/core/tests/switch_core/retention/test_service.pyindex 548bbbaea..fb8874684 100644--- a/core/tests/switch_core/retention/test_service.py+++ b/core/tests/switch_core/retention/test_service.py@@ -16,9 +16,11 @@ import pytestfrom sqlalchemy import func, selectfrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker+from switch_core.attachments import ATTACHMENT_GROUP_KEYfrom switch_core.db.models import (TENANT_ZERO_ID,Agent,+ AgentEventRow,ApiKey,ApprovalRequest,BridgeMessageMap,@@ -212,6 +214,57 @@ class TestMessages:assert attachments == 0assert mapped == {"$unrelated"}+ async def test_agents_copies_go_with_their_message_files_sent_together_too(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ room = await _room(session, "copies")+ for event_id in ("$part-1", "$part-2"):+ await MessageStore().create(+ session,+ Message(+ room_id=room.id,+ transport_event_id=event_id,+ sender_id="@sender:test",+ event_type="m.room.message",+ msgtype="m.file",+ body=event_id,+ content={+ "msgtype": "m.file",+ "body": event_id,+ ATTACHMENT_GROUP_KEY: {"id": "group-1", "count": 2},+ },+ sent_at=NOW - timedelta(days=60),+ ),+ [],+ )+ await _message(session, room, "$old-single", age_days=60)+ recent = await _message(session, room, "$recent", age_days=1)+ agent = await _agent(session)+ for seq, key in enumerate(("group-1", "$old-single", "$recent"), start=1):+ session.add(+ AgentEventRow(+ agent_id=agent.id,+ seq=seq,+ room_id=room.id,+ room_seq=recent.seq,+ message_key=key,+ type="message",+ payload={"body": key},+ notifiable=True,+ )+ )+ await session.commit()+ await _set_policy(session_factory, 30, TENANT_ZERO_ID)++ await RetentionService(session_factory).apply(NOW, BUDGET)++ async with session_factory() as session:+ kept = set(+ (await session.execute(select(AgentEventRow.message_key))).scalars()+ )+ assert kept == {"$recent"}+async def test_another_workspaces_window_does_not_reach_this_one(self, session_factory: async_sessionmaker[AsyncSession]) -> None: