Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: add bulk deletion for passages (#2489)
cthomas71f42473a91b
diff --git a/letta/orm/sqlalchemy_base.py b/letta/orm/sqlalchemy_base.pyindex f3ec891b5..88affc71a 100644--- a/letta/orm/sqlalchemy_base.py+++ b/letta/orm/sqlalchemy_base.py@@ -4,7 +4,7 @@ from functools import wrapsfrom pprint import pformatfrom typing import TYPE_CHECKING, List, Literal, Optional, Tuple, Union-from sqlalchemy import String, and_, func, or_, select+from sqlalchemy import String, and_, delete, func, or_, selectfrom sqlalchemy.exc import DBAPIError, IntegrityError, TimeoutErrorfrom sqlalchemy.ext.asyncio import AsyncSessionfrom sqlalchemy.orm import Mapped, Session, mapped_column@@ -763,6 +763,36 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):logger.exception(f"Failed to hard delete {self.__class__.__name__} with ID {self.id}")raise ValueError(f"Failed to hard delete {self.__class__.__name__} with ID {self.id}: {e}")+ @classmethod+ @handle_db_timeout+ async def bulk_hard_delete_async(+ cls,+ db_session: "AsyncSession",+ identifiers: List[str],+ actor: Optional["User"],+ access: Optional[List[Literal["read", "write", "admin"]]] = ["write"],+ access_type: AccessType = AccessType.ORGANIZATION,+ ) -> None:+ """Permanently removes the record from the database asynchronously."""+ logger.debug(f"Hard deleting {cls.__name__} with IDs: {identifiers} with actor={actor} (async)")++ if len(identifiers) == 0:+ logger.debug(f"No identifiers provided for {cls.__name__}, nothing to delete")+ return++ query = delete(cls)+ query = query.where(cls.id.in_(identifiers))+ query = cls.apply_access_predicate(query, actor, access, access_type)+ async with db_session as session:+ try:+ result = await session.execute(query)+ await session.commit()+ logger.debug(f"Successfully deleted {result.rowcount} {cls.__name__} records")+ except Exception as e:+ await session.rollback()+ logger.exception(f"Failed to hard delete {cls.__name__} with identifiers {identifiers}")+ raise ValueError(f"Failed to hard delete {cls.__name__} with identifiers {identifiers}: {e}")+@handle_db_timeoutdef update(self, db_session: Session, actor: Optional["User"] = None, no_commit: bool = False) -> "SqlalchemyBase":logger.debug(...)diff --git a/letta/server/rest_api/routers/v1/agents.py b/letta/server/rest_api/routers/v1/agents.pyindex 6cc759630..11f9f69cb 100644--- a/letta/server/rest_api/routers/v1/agents.py+++ b/letta/server/rest_api/routers/v1/agents.py@@ -330,9 +330,9 @@ async def detach_source(agent = await server.agent_manager.detach_source_async(agent_id=agent_id, source_id=source_id, actor=actor)if agent.enable_sleeptime:try:- source = server.source_manager.get_source_by_id(source_id=source_id)- block = server.agent_manager.get_block_with_label(agent_id=agent.id, block_label=source.name, actor=actor)- server.block_manager.delete_block(block.id, actor)+ source = await server.source_manager.get_source_by_id(source_id=source_id)+ block = await server.agent_manager.get_block_with_label_async(agent_id=agent.id, block_label=source.name, actor=actor)+ await server.block_manager.delete_block_async(block.id, actor)except:passreturn agentdiff --git a/letta/server/server.py b/letta/server/server.pyindex aeae33920..02c9f196a 100644--- a/letta/server/server.py+++ b/letta/server/server.py@@ -19,6 +19,7 @@ import letta.constants as constantsimport letta.server.utils as server_utilsimport letta.system as systemfrom letta.agent import Agent, save_agent+from letta.agents.letta_agent import LettaAgentfrom letta.config import LettaConfigfrom letta.constants import LETTA_TOOL_EXECUTION_DIRfrom letta.data_sources.connectors import DataConnector, load_data@@ -1295,11 +1296,8 @@ class SyncServer(Server):await self.source_manager.delete_source(source_id=source_id, actor=actor)# delete data from passage store- # TODO: make async- passages_to_be_deleted = self.agent_manager.list_passages(actor=actor, source_id=source_id, limit=None)-- # TODO: make this async- self.passage_manager.delete_passages(actor=actor, passages=passages_to_be_deleted)+ passages_to_be_deleted = await self.agent_manager.list_passages_async(actor=actor, source_id=source_id, limit=None)+ await self.passage_manager.delete_source_passages_async(actor=actor, passages=passages_to_be_deleted)# TODO: delete data from agent passage stores (?)@@ -1345,9 +1343,9 @@ class SyncServer(Server):async def sleeptime_document_ingest_async(self, main_agent: AgentState, source: Source, actor: User, clear_history: bool = False) -> None:- sleeptime_agent = await self.create_document_sleeptime_agent_async(main_agent, source, actor, clear_history)+ sleeptime_agent_state = await self.create_document_sleeptime_agent_async(main_agent, source, actor, clear_history)sleeptime_agent = LettaAgent(- agent_id=sleeptime_agent.id,+ agent_id=sleeptime_agent_state.id,message_manager=self.message_manager,agent_manager=self.agent_manager,block_manager=self.block_manager,@@ -1363,7 +1361,7 @@ class SyncServer(Server):MessageCreate(role="user", content=passage.text),])- await self.agent_manager.delete_agent_async(agent_id=sleeptime_agent.id, actor=actor)+ await self.agent_manager.delete_agent_async(agent_id=sleeptime_agent_state.id, actor=actor)async def create_document_sleeptime_agent_async(self, main_agent: AgentState, source: Source, actor: User, clear_history: bool = Falsediff --git a/letta/services/agent_manager.py b/letta/services/agent_manager.pyindex 5e36b9e6a..95e721d0a 100644--- a/letta/services/agent_manager.py+++ b/letta/services/agent_manager.py@@ -1658,9 +1658,8 @@ class AgentManager:if not block_ids:return agent_state- agent_state.memory.blocks = await self.block_manager.get_all_blocks_by_ids_async(- block_ids=[b.id for b in agent_state.memory.blocks], actor=actor- )+ blocks = await self.block_manager.get_all_blocks_by_ids_async(block_ids=[b.id for b in agent_state.memory.blocks], actor=actor)+ agent_state.memory.blocks = [b for b in blocks if b is not None]return agent_state# ======================================================================================================================diff --git a/letta/services/passage_manager.py b/letta/services/passage_manager.pyindex 9edffcedc..4b233ddb3 100644--- a/letta/services/passage_manager.py+++ b/letta/services/passage_manager.py@@ -383,6 +383,17 @@ class PassageManager:self.delete_passage_by_id(passage_id=passage.id, actor=actor)return True+ @enforce_types+ @trace_method+ async def delete_source_passages_async(+ self,+ actor: PydanticUser,+ passages: List[PydanticPassage],+ ) -> bool:+ async with db_registry.async_session() as session:+ await SourcePassage.bulk_hard_delete_async(db_session=session, identifiers=[p.id for p in passages], actor=actor)+ return True+@enforce_types@trace_methoddef size(