Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Delete agents' copies of deleted messages by message key, in retention too

Petr Bauche240b9e14e08
diff --git a/core/switch_core/db/stores/erasure_store.py b/core/switch_core/db/stores/erasure_store.py
index 1d21c28ba..dbe7dfe1b 100644
--- a/core/switch_core/db/stores/erasure_store.py
+++ b/core/switch_core/db/stores/erasure_store.py
@@ -38,7 +38,6 @@ from sqlalchemy.ext.asyncio import AsyncSession
from switch_core.chats import MEMBER_CLIENT_TYPE
from switch_core.db.models import (
- AgentEventRow,
ApprovalRequest,
Client,
ClientRoom,
@@ -496,21 +495,6 @@ class ErasureStore:
)
return result.scalar_one()
- async def delete_agent_copies(
- self, session: AsyncSession, transport_event_ids: Collection[str]
- ) -> None:
- """Delete the copies of these (deleted) messages stored as events for
- agents (`agent_events`), which would otherwise outlive them until the
- event retention sweep."""
- if not transport_event_ids:
- return
- await session.execute(
- delete(AgentEventRow).where(
- AgentEventRow.tenant_id == require_tenant_id(),
- AgentEventRow.message_key.in_(list(transport_event_ids)),
- )
- )
-
async def scrub_approval_answers(
self, session: AsyncSession, transport_user_id: str
) -> int:
diff --git a/core/switch_core/db/stores/message_store.py b/core/switch_core/db/stores/message_store.py
index b6ccb7857..4845b2e09 100644
--- a/core/switch_core/db/stores/message_store.py
+++ b/core/switch_core/db/stores/message_store.py
@@ -14,6 +14,7 @@ from sqlalchemy import (
)
from sqlalchemy.ext.asyncio import AsyncSession
+from switch_core.attachments import ATTACHMENT_GROUP_KEY
from switch_core.db.id_lists import any_of
from switch_core.db.models import Message, MessageAttachment, Room, require_tenant_id
from switch_core.db.notify_ddl import SET_WRITER_ANNOUNCES
@@ -23,6 +24,9 @@ from switch_core.db.raw import RawStatement
@dataclass(frozen=True)
class DeletedMessages:
event_ids: list[str]
+ #: The keys agents' copies of them are stored under (`agent_events.message_key`):
+ #: the attachment group id for a file sent as part of several, else the event id.
+ message_keys: set[str]
#: The files the deleted messages carried. Others may still quote them.
uris: set[str]
#: The rooms they were deleted from.
@@ -220,16 +224,13 @@ class MessageStore:
async def delete_sent_before(
self, session: AsyncSession, cutoff: datetime, *, limit: int
- ) -> list[str]:
+ ) -> DeletedMessages:
"""Delete up to `limit` of the bound tenant's oldest messages sent before
- `cutoff`, with their attachments. Returns their transport event ids.
+ `cutoff`, with their attachments.
See `_delete_where` for what else changes and what does not.
"""
- deleted = await self._delete_where(
- session, Message.sent_at < cutoff, limit=limit
- )
- return deleted.event_ids
+ return await self._delete_where(session, Message.sent_at < cutoff, limit=limit)
async def delete_sent_by(
self, session: AsyncSession, *, transport_user_id: str, limit: int
@@ -262,7 +263,14 @@ class MessageStore:
rows = (
await session.execute(
select(
- Message.id, Message.room_id, Message.seq, Message.transport_event_id
+ Message.id,
+ Message.room_id,
+ Message.seq,
+ Message.transport_event_id,
+ func.coalesce(
+ Message.content[ATTACHMENT_GROUP_KEY]["id"].astext,
+ Message.transport_event_id,
+ ).label("message_key"),
)
.where(Message.tenant_id == require_tenant_id(), condition)
.order_by(Message.sent_at)
@@ -270,7 +278,9 @@ class MessageStore:
)
).all()
if not rows:
- return DeletedMessages(event_ids=[], uris=set(), room_ids=set())
+ return DeletedMessages(
+ event_ids=[], message_keys=set(), uris=set(), room_ids=set()
+ )
message_ids = [r.id for r in rows]
uris = set(
(
@@ -294,6 +304,7 @@ class MessageStore:
await session.execute(delete(Message).where(Message.id.in_(message_ids)))
return DeletedMessages(
event_ids=[row.transport_event_id for row in rows],
+ message_keys={row.message_key for row in rows},
uris=uris,
room_ids=set(floors),
)
diff --git a/core/switch_core/db/stores/retention_store.py b/core/switch_core/db/stores/retention_store.py
index 3e6e61178..5d5f0b642 100644
--- a/core/switch_core/db/stores/retention_store.py
+++ b/core/switch_core/db/stores/retention_store.py
@@ -16,6 +16,7 @@ from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession
from switch_core.db.models import (
+ AgentEventRow,
ApprovalRequest,
BridgeMessageMap,
Invitation,
@@ -119,6 +120,21 @@ class RetentionStore:
)
return _rowcount(result)
+ async def delete_agent_copies(
+ self, session: AsyncSession, message_keys: Collection[str]
+ ) -> None:
+ """Delete the copies of these (deleted) messages stored as events for
+ agents (`agent_events`), keyed as `DeletedMessages.message_keys`, which
+ would otherwise outlive them until the event retention sweep."""
+ if not message_keys:
+ return
+ await session.execute(
+ delete(AgentEventRow).where(
+ AgentEventRow.tenant_id == require_tenant_id(),
+ AgentEventRow.message_key.in_(list(message_keys)),
+ )
+ )
+
async def delete_unreferenced_media(
self, session: AsyncSession, *, created_before: datetime, limit: int
) -> int:
diff --git a/core/switch_core/retention/erasure.py b/core/switch_core/retention/erasure.py
index d3e57ea18..8e6d45511 100644
--- a/core/switch_core/retention/erasure.py
+++ b/core/switch_core/retention/erasure.py
@@ -299,7 +299,7 @@ class ErasureService:
db, transport_user_id=transport_user_id, limit=MESSAGE_BATCH
)
await self._retention.delete_bridge_mappings(db, deleted.event_ids)
- await self._erasures.delete_agent_copies(db, deleted.event_ids)
+ await self._retention.delete_agent_copies(db, deleted.message_keys)
await self._erasures.add_progress(
db, erasure_id, messages=len(deleted.event_ids), files=0, identities=0
)
diff --git a/core/switch_core/retention/service.py b/core/switch_core/retention/service.py
index 88d5b014f..3a6ba4677 100644
--- a/core/switch_core/retention/service.py
+++ b/core/switch_core/retention/service.py
@@ -154,12 +154,13 @@ class RetentionService:
deleted = 0
while True:
async with tenant_session(self._sessions, tenant_id) as db, db.begin():
- event_ids = await self._messages.delete_sent_before(
+ batch = await self._messages.delete_sent_before(
db, cutoff, limit=MESSAGE_BATCH
)
- await self._retention.delete_bridge_mappings(db, event_ids)
- deleted += len(event_ids)
- if len(event_ids) < MESSAGE_BATCH:
+ await self._retention.delete_bridge_mappings(db, batch.event_ids)
+ await self._retention.delete_agent_copies(db, batch.message_keys)
+ deleted += len(batch.event_ids)
+ if len(batch.event_ids) < MESSAGE_BATCH:
return deleted, False
if time.monotonic() >= deadline:
return deleted, True
diff --git a/core/tests/switch_core/retention/test_erasure.py b/core/tests/switch_core/retention/test_erasure.py
index ccf03d252..1b98ce224 100644
--- a/core/tests/switch_core/retention/test_erasure.py
+++ b/core/tests/switch_core/retention/test_erasure.py
@@ -15,6 +15,7 @@ import pytest
from sqlalchemy import func, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+from switch_core.attachments import ATTACHMENT_GROUP_KEY
from switch_core.db.models import (
Agent,
AgentEventRow,
@@ -550,6 +551,62 @@ class TestErasing:
assert channel_after.name == "Acme Slack: ana-fans"
assert copies == 0
+ async def test_agent_copies_of_files_sent_together_go(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ async with session_factory() as session:
+ person = await _person(session, await _bridge(session), "ana")
+ client = await _client_of(session, person)
+ room = await _room(session, "files")
+ for event_id in ("$part-1", "$part-2"):
+ part = await MessageStore().create(
+ session,
+ Message(
+ room_id=room.id,
+ transport_event_id=event_id,
+ sender_id=client.transport_user_id,
+ sender_client_id=client.id,
+ event_type="m.room.message",
+ msgtype="m.file",
+ body=event_id,
+ content={
+ "msgtype": "m.file",
+ "body": event_id,
+ ATTACHMENT_GROUP_KEY: {"id": "group-ana", "count": 2},
+ },
+ ),
+ [],
+ )
+ other = await _person(session, await _bridge(session), "bo")
+ kept = await _say(session, room, await _client_of(session, other), "$bo")
+ agent = await _agent(session)
+ for seq, key in enumerate(("group-ana", kept.transport_event_id), start=1):
+ session.add(
+ AgentEventRow(
+ agent_id=agent.id,
+ seq=seq,
+ room_id=room.id,
+ room_seq=part.seq,
+ message_key=key,
+ type="message",
+ payload={"body": key},
+ notifiable=True,
+ )
+ )
+ await session.commit()
+ person_id = person.id
+ await _queue(session_factory, [person_id])
+
+ await ErasureService(session_factory, _Clients(), _Bridges()).work_once(
+ datetime.now(UTC)
+ )
+
+ async with session_factory() as session:
+ copies = set(
+ (await session.execute(select(AgentEventRow.message_key))).scalars()
+ )
+ assert copies == {"$bo"}
+
class TestQueue:
async def test_people_are_listed_with_their_message_counts_and_claimants(
diff --git a/core/tests/switch_core/retention/test_service.py b/core/tests/switch_core/retention/test_service.py
index 548bbbaea..fb8874684 100644
--- a/core/tests/switch_core/retention/test_service.py
+++ b/core/tests/switch_core/retention/test_service.py
@@ -16,9 +16,11 @@ import pytest
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+from switch_core.attachments import ATTACHMENT_GROUP_KEY
from switch_core.db.models import (
TENANT_ZERO_ID,
Agent,
+ AgentEventRow,
ApiKey,
ApprovalRequest,
BridgeMessageMap,
@@ -212,6 +214,57 @@ class TestMessages:
assert attachments == 0
assert mapped == {"$unrelated"}
+ async def test_agents_copies_go_with_their_message_files_sent_together_too(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ async with session_factory() as session:
+ room = await _room(session, "copies")
+ for event_id in ("$part-1", "$part-2"):
+ await MessageStore().create(
+ session,
+ Message(
+ room_id=room.id,
+ transport_event_id=event_id,
+ sender_id="@sender:test",
+ event_type="m.room.message",
+ msgtype="m.file",
+ body=event_id,
+ content={
+ "msgtype": "m.file",
+ "body": event_id,
+ ATTACHMENT_GROUP_KEY: {"id": "group-1", "count": 2},
+ },
+ sent_at=NOW - timedelta(days=60),
+ ),
+ [],
+ )
+ await _message(session, room, "$old-single", age_days=60)
+ recent = await _message(session, room, "$recent", age_days=1)
+ agent = await _agent(session)
+ for seq, key in enumerate(("group-1", "$old-single", "$recent"), start=1):
+ session.add(
+ AgentEventRow(
+ agent_id=agent.id,
+ seq=seq,
+ room_id=room.id,
+ room_seq=recent.seq,
+ message_key=key,
+ type="message",
+ payload={"body": key},
+ notifiable=True,
+ )
+ )
+ await session.commit()
+ await _set_policy(session_factory, 30, TENANT_ZERO_ID)
+
+ await RetentionService(session_factory).apply(NOW, BUDGET)
+
+ async with session_factory() as session:
+ kept = set(
+ (await session.execute(select(AgentEventRow.message_key))).scalars()
+ )
+ assert kept == {"$recent"}
+
async def test_another_workspaces_window_does_not_reach_this_one(
self, session_factory: async_sessionmaker[AsyncSession]
) -> None: