Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
chore: create file manager (#2693)
cthomasc300e58db973
diff --git a/letta/data_sources/connectors.py b/letta/data_sources/connectors.pyindex ce12d5733..3248e83ca 100644--- a/letta/data_sources/connectors.py+++ b/letta/data_sources/connectors.py@@ -8,8 +8,8 @@ from letta.embeddings import embedding_modelfrom letta.schemas.file import FileMetadatafrom letta.schemas.passage import Passagefrom letta.schemas.source import Source+from letta.services.file_manager import FileManagerfrom letta.services.passage_manager import PassageManager-from letta.services.source_manager import SourceManagerclass DataConnector:@@ -38,9 +38,7 @@ class DataConnector:"""-async def load_data(- connector: DataConnector, source: Source, passage_manager: PassageManager, source_manager: SourceManager, actor: "User"-):+async def load_data(connector: DataConnector, source: Source, passage_manager: PassageManager, file_manager: FileManager, actor: "User"):from letta.llm_api.llm_client import LLMClientfrom letta.schemas.embedding_config import EmbeddingConfig@@ -94,7 +92,7 @@ async def load_data(for file_metadata in connector.find_files(source):file_count += 1- await source_manager.create_file(file_metadata, actor)+ await file_manager.create_file(file_metadata, actor)# generate passagesfor passage_text, passage_metadata in connector.generate_passages(file_metadata, chunk_size=embedding_config.embedding_chunk_size):diff --git a/letta/server/rest_api/routers/v1/agents.py b/letta/server/rest_api/routers/v1/agents.pyindex b7dfeb9a3..7a6d71633 100644--- a/letta/server/rest_api/routers/v1/agents.py+++ b/letta/server/rest_api/routers/v1/agents.py@@ -316,7 +316,7 @@ async def attach_source(# Check if the agent is missing any files toolsagent_state = await server.agent_manager.attach_missing_files_tools_async(agent_state=agent_state, actor=actor)- files = await server.source_manager.list_files(source_id, actor, include_content=True)+ files = await server.file_manager.list_files(source_id, actor, include_content=True)texts = []file_ids = []file_names = []@@ -354,7 +354,7 @@ async def detach_source(if not agent_state.sources:agent_state = await server.agent_manager.detach_all_files_tools_async(agent_state=agent_state, actor=actor)- files = await server.source_manager.list_files(source_id, actor)+ files = await server.file_manager.list_files(source_id, actor)file_ids = [f.id for f in files]await server.remove_files_from_context_window(agent_state=agent_state, file_ids=file_ids, actor=actor)diff --git a/letta/server/rest_api/routers/v1/sources.py b/letta/server/rest_api/routers/v1/sources.pyindex 2cdf9090f..0ff504f9f 100644--- a/letta/server/rest_api/routers/v1/sources.py+++ b/letta/server/rest_api/routers/v1/sources.py@@ -154,7 +154,7 @@ async def delete_source(actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)source = await server.source_manager.get_source_by_id(source_id=source_id, actor=actor)agent_states = await server.source_manager.list_attached_agents(source_id=source_id, actor=actor)- files = await server.source_manager.list_files(source_id, actor)+ files = await server.file_manager.list_files(source_id, actor)file_ids = [f.id for f in files]for agent_state in agent_states:@@ -294,7 +294,7 @@ async def list_source_files(List paginated files associated with a data source."""actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)- return await server.source_manager.list_files(+ return await server.file_manager.list_files(source_id=source_id,limit=limit,after=after,@@ -317,7 +317,7 @@ async def delete_file_from_source("""actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)- deleted_file = await server.source_manager.delete_file(file_id=file_id, actor=actor)+ deleted_file = await server.file_manager.delete_file(file_id=file_id, actor=actor)await server.remove_file_from_context_windows(source_id=source_id, file_id=deleted_file.id, actor=actor)diff --git a/letta/server/server.py b/letta/server/server.pyindex 1ee944d3c..400505f13 100644--- a/letta/server/server.py+++ b/letta/server/server.py@@ -80,6 +80,7 @@ from letta.server.rest_api.interface import StreamingServerInterfacefrom letta.server.rest_api.utils import sse_async_generatorfrom letta.services.agent_manager import AgentManagerfrom letta.services.block_manager import BlockManager+from letta.services.file_manager import FileManagerfrom letta.services.files_agents_manager import FileAgentManagerfrom letta.services.group_manager import GroupManagerfrom letta.services.helpers.tool_execution_helper import prepare_local_sandbox@@ -219,6 +220,7 @@ class SyncServer(Server):self.batch_manager = LLMBatchManager()self.telemetry_manager = TelemetryManager()self.file_agent_manager = FileAgentManager()+ self.file_manager = FileManager()# A resusable httpx clienttimeout = httpx.Timeout(connect=10.0, read=20.0, write=10.0, pool=10.0)@@ -1507,7 +1509,7 @@ class SyncServer(Server):raise ValueError(f"Data source {source_name} does not exist for user {user_id}")# load data into the document store- passage_count, document_count = await load_data(connector, source, self.passage_manager, self.source_manager, actor=actor)+ passage_count, document_count = await load_data(connector, source, self.passage_manager, self.file_manager, actor=actor)return passage_count, document_countdef list_data_source_passages(self, user_id: str, source_id: str) -> List[Passage]:diff --git a/letta/services/file_manager.py b/letta/services/file_manager.pynew file mode 100644index 000000000..ee43dd54b--- /dev/null+++ b/letta/services/file_manager.py@@ -0,0 +1,219 @@+from datetime import datetime+from typing import List, Optional++from sqlalchemy import select, update+from sqlalchemy.dialects.postgresql import insert as pg_insert+from sqlalchemy.exc import IntegrityError+from sqlalchemy.orm import selectinload++from letta.orm.errors import NoResultFound+from letta.orm.file import FileContent as FileContentModel+from letta.orm.file import FileMetadata as FileMetadataModel+from letta.orm.sqlalchemy_base import AccessType+from letta.otel.tracing import trace_method+from letta.schemas.enums import FileProcessingStatus+from letta.schemas.file import FileMetadata as PydanticFileMetadata+from letta.schemas.user import User as PydanticUser+from letta.server.db import db_registry+from letta.utils import enforce_types+++class FileManager:+ """Manager class to handle business logic related to files."""++ @enforce_types+ @trace_method+ async def create_file(+ self,+ file_metadata: PydanticFileMetadata,+ actor: PydanticUser,+ *,+ text: Optional[str] = None,+ ) -> PydanticFileMetadata:++ # short-circuit if it already exists+ existing = await self.get_file_by_id(file_metadata.id, actor=actor)+ if existing:+ return existing++ async with db_registry.async_session() as session:+ try:+ file_metadata.organization_id = actor.organization_id+ file_orm = FileMetadataModel(**file_metadata.model_dump(to_orm=True, exclude_none=True))+ await file_orm.create_async(session, actor=actor, no_commit=True)++ if text is not None:+ content_orm = FileContentModel(file_id=file_orm.id, text=text)+ await content_orm.create_async(session, actor=actor, no_commit=True)++ await session.commit()+ await session.refresh(file_orm)+ return await file_orm.to_pydantic_async()++ except IntegrityError:+ await session.rollback()+ return await self.get_file_by_id(file_metadata.id, actor=actor)++ # TODO: We make actor optional for now, but should most likely be enforced due to security reasons+ @enforce_types+ @trace_method+ async def get_file_by_id(+ self,+ file_id: str,+ actor: Optional[PydanticUser] = None,+ *,+ include_content: bool = False,+ ) -> Optional[PydanticFileMetadata]:+ """Retrieve a file by its ID.++ If `include_content=True`, the FileContent relationship is eagerly+ loaded so `to_pydantic(include_content=True)` never triggers a+ lazy SELECT (avoids MissingGreenlet).+ """+ async with db_registry.async_session() as session:+ try:+ if include_content:+ # explicit eager load+ query = (+ select(FileMetadataModel).where(FileMetadataModel.id == file_id).options(selectinload(FileMetadataModel.content))+ )+ # apply org-scoping if actor provided+ if actor:+ query = FileMetadataModel.apply_access_predicate(+ query,+ actor,+ access=["read"],+ access_type=AccessType.ORGANIZATION,+ )++ result = await session.execute(query)+ file_orm = result.scalar_one()+ else:+ # fast path (metadata only)+ file_orm = await FileMetadataModel.read_async(+ db_session=session,+ identifier=file_id,+ actor=actor,+ )++ return await file_orm.to_pydantic_async(include_content=include_content)++ except NoResultFound:+ return None++ @enforce_types+ @trace_method+ async def update_file_status(+ self,+ *,+ file_id: str,+ actor: PydanticUser,+ processing_status: Optional[FileProcessingStatus] = None,+ error_message: Optional[str] = None,+ ) -> PydanticFileMetadata:+ """+ Update processing_status and/or error_message on a FileMetadata row.++ * 1st round-trip → UPDATE+ * 2nd round-trip → SELECT fresh row (same as read_async)+ """++ if processing_status is None and error_message is None:+ raise ValueError("Nothing to update")++ values: dict[str, object] = {"updated_at": datetime.utcnow()}+ if processing_status is not None:+ values["processing_status"] = processing_status+ if error_message is not None:+ values["error_message"] = error_message++ async with db_registry.async_session() as session:+ # Fast in-place update – no ORM hydration+ stmt = (+ update(FileMetadataModel)+ .where(+ FileMetadataModel.id == file_id,+ FileMetadataModel.organization_id == actor.organization_id,+ )+ .values(**values)+ )+ await session.execute(stmt)+ await session.commit()++ # Reload via normal accessor so we return a fully-attached object+ file_orm = await FileMetadataModel.read_async(+ db_session=session,+ identifier=file_id,+ actor=actor,+ )+ return await file_orm.to_pydantic_async()++ @enforce_types+ @trace_method+ async def upsert_file_content(+ self,+ *,+ file_id: str,+ text: str,+ actor: PydanticUser,+ ) -> PydanticFileMetadata:+ async with db_registry.async_session() as session:+ await FileMetadataModel.read_async(session, file_id, actor)++ dialect_name = session.bind.dialect.name++ if dialect_name == "postgresql":+ stmt = (+ pg_insert(FileContentModel)+ .values(file_id=file_id, text=text)+ .on_conflict_do_update(+ index_elements=[FileContentModel.file_id],+ set_={"text": text},+ )+ )+ await session.execute(stmt)+ else:+ # Emulate upsert for SQLite and others+ stmt = select(FileContentModel).where(FileContentModel.file_id == file_id)+ result = await session.execute(stmt)+ existing = result.scalar_one_or_none()++ if existing:+ await session.execute(update(FileContentModel).where(FileContentModel.file_id == file_id).values(text=text))+ else:+ session.add(FileContentModel(file_id=file_id, text=text))++ await session.commit()++ # Reload with content+ query = select(FileMetadataModel).options(selectinload(FileMetadataModel.content)).where(FileMetadataModel.id == file_id)+ result = await session.execute(query)+ return await result.scalar_one().to_pydantic_async(include_content=True)++ @enforce_types+ @trace_method+ async def list_files(+ self, source_id: str, actor: PydanticUser, after: Optional[str] = None, limit: Optional[int] = 50, include_content: bool = False+ ) -> List[PydanticFileMetadata]:+ """List all files with optional pagination."""+ async with db_registry.async_session() as session:+ options = [selectinload(FileMetadataModel.content)] if include_content else None++ files = await FileMetadataModel.list_async(+ db_session=session,+ after=after,+ limit=limit,+ organization_id=actor.organization_id,+ source_id=source_id,+ query_options=options,+ )+ return [await file.to_pydantic_async(include_content=include_content) for file in files]++ @enforce_types+ @trace_method+ async def delete_file(self, file_id: str, actor: PydanticUser) -> PydanticFileMetadata:+ """Delete a file by its ID."""+ async with db_registry.async_session() as session:+ file = await FileMetadataModel.read_async(db_session=session, identifier=file_id)+ await file.hard_delete_async(db_session=session, actor=actor)+ return await file.to_pydantic_async()diff --git a/letta/services/file_processor/file_processor.py b/letta/services/file_processor/file_processor.pyindex c82786cc7..20e2bf50e 100644--- a/letta/services/file_processor/file_processor.py+++ b/letta/services/file_processor/file_processor.py@@ -11,6 +11,7 @@ from letta.schemas.job import Job, JobUpdatefrom letta.schemas.passage import Passagefrom letta.schemas.user import Userfrom letta.server.server import SyncServer+from letta.services.file_manager import FileManagerfrom letta.services.file_processor.chunker.line_chunker import LineChunkerfrom letta.services.file_processor.chunker.llama_index_chunker import LlamaIndexChunkerfrom letta.services.file_processor.embedder.openai_embedder import OpenAIEmbedder@@ -38,6 +39,7 @@ class FileProcessor:self.line_chunker = LineChunker()self.embedder = embedderself.max_file_size = max_file_size+ self.file_manager = FileManager()self.source_manager = SourceManager()self.passage_manager = PassageManager()self.job_manager = JobManager()@@ -58,7 +60,7 @@ class FileProcessor:# Create file as early as possible with no contentfile_metadata.processing_status = FileProcessingStatus.PARSING # Parsing now- file_metadata = await self.source_manager.create_file(file_metadata, self.actor)+ file_metadata = await self.file_manager.create_file(file_metadata, self.actor)try:# Ensure we're working with bytes@@ -73,10 +75,8 @@ class FileProcessor:# update file with raw textraw_markdown_text = "".join([page.markdown for page in ocr_response.pages])- file_metadata = await self.source_manager.upsert_file_content(- file_id=file_metadata.id, text=raw_markdown_text, actor=self.actor- )- file_metadata = await self.source_manager.update_file_status(+ file_metadata = await self.file_manager.upsert_file_content(file_id=file_metadata.id, text=raw_markdown_text, actor=self.actor)+ file_metadata = await self.file_manager.update_file_status(file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.EMBEDDING)@@ -123,7 +123,7 @@ class FileProcessor:job.metadata["num_passages"] = len(all_passages)await self.job_manager.update_job_by_id_async(job_id=job.id, job_update=JobUpdate(**job.model_dump()), actor=self.actor)- await self.source_manager.update_file_status(+ await self.file_manager.update_file_status(file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.COMPLETED)@@ -138,7 +138,7 @@ class FileProcessor:job.metadata["error"] = str(e)await self.job_manager.update_job_by_id_async(job_id=job.id, job_update=JobUpdate(**job.model_dump()), actor=self.actor)- await self.source_manager.update_file_status(+ await self.file_manager.update_file_status(file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.ERROR, error_message=str(e))diff --git a/letta/services/source_manager.py b/letta/services/source_manager.pyindex 614c0c31c..88aa4c6d3 100644--- a/letta/services/source_manager.py+++ b/letta/services/source_manager.py@@ -1,21 +1,10 @@import asyncio-from datetime import datetimefrom typing import List, Optional-from sqlalchemy import select, update-from sqlalchemy.dialects.postgresql import insert as pg_insert-from sqlalchemy.exc import IntegrityError-from sqlalchemy.orm import selectinload-from letta.orm.errors import NoResultFound-from letta.orm.file import FileContent as FileContentModel-from letta.orm.file import FileMetadata as FileMetadataModelfrom letta.orm.source import Source as SourceModel-from letta.orm.sqlalchemy_base import AccessTypefrom letta.otel.tracing import trace_methodfrom letta.schemas.agent import AgentState as PydanticAgentState-from letta.schemas.enums import FileProcessingStatus-from letta.schemas.file import FileMetadata as PydanticFileMetadatafrom letta.schemas.source import Source as PydanticSourcefrom letta.schemas.source import SourceUpdatefrom letta.schemas.user import User as PydanticUser@@ -148,200 +137,3 @@ class SourceManager:return Noneelse:return sources[0].to_pydantic()-- @enforce_types- @trace_method- async def create_file(- self,- file_metadata: PydanticFileMetadata,- actor: PydanticUser,- *,- text: Optional[str] = None,- ) -> PydanticFileMetadata:-- # short-circuit if it already exists- existing = await self.get_file_by_id(file_metadata.id, actor=actor)- if existing:- return existing-- async with db_registry.async_session() as session:- try:- file_metadata.organization_id = actor.organization_id- file_orm = FileMetadataModel(**file_metadata.model_dump(to_orm=True, exclude_none=True))- await file_orm.create_async(session, actor=actor, no_commit=True)-- if text is not None:- content_orm = FileContentModel(file_id=file_orm.id, text=text)- await content_orm.create_async(session, actor=actor, no_commit=True)-- await session.commit()- await session.refresh(file_orm)- return await file_orm.to_pydantic_async()-- except IntegrityError:- await session.rollback()- return await self.get_file_by_id(file_metadata.id, actor=actor)-- # TODO: We make actor optional for now, but should most likely be enforced due to security reasons- @enforce_types- @trace_method- async def get_file_by_id(- self,- file_id: str,- actor: Optional[PydanticUser] = None,- *,- include_content: bool = False,- ) -> Optional[PydanticFileMetadata]:- """Retrieve a file by its ID.-- If `include_content=True`, the FileContent relationship is eagerly- loaded so `to_pydantic(include_content=True)` never triggers a- lazy SELECT (avoids MissingGreenlet).- """- async with db_registry.async_session() as session:- try:- if include_content:- # explicit eager load- query = (- select(FileMetadataModel).where(FileMetadataModel.id == file_id).options(selectinload(FileMetadataModel.content))- )- # apply org-scoping if actor provided- if actor:- query = FileMetadataModel.apply_access_predicate(- query,- actor,- access=["read"],- access_type=AccessType.ORGANIZATION,- )-- result = await session.execute(query)- file_orm = result.scalar_one()- else:- # fast path (metadata only)- file_orm = await FileMetadataModel.read_async(- db_session=session,- identifier=file_id,- actor=actor,- )-- return await file_orm.to_pydantic_async(include_content=include_content)-- except NoResultFound:- return None-- @enforce_types- @trace_method- async def update_file_status(- self,- *,- file_id: str,- actor: PydanticUser,- processing_status: Optional[FileProcessingStatus] = None,- error_message: Optional[str] = None,- ) -> PydanticFileMetadata:- """- Update processing_status and/or error_message on a FileMetadata row.-- * 1st round-trip → UPDATE- * 2nd round-trip → SELECT fresh row (same as read_async)- """-- if processing_status is None and error_message is None:- raise ValueError("Nothing to update")-- values: dict[str, object] = {"updated_at": datetime.utcnow()}- if processing_status is not None:- values["processing_status"] = processing_status- if error_message is not None:- values["error_message"] = error_message-- async with db_registry.async_session() as session:- # Fast in-place update – no ORM hydration- stmt = (- update(FileMetadataModel)- .where(- FileMetadataModel.id == file_id,- FileMetadataModel.organization_id == actor.organization_id,- )- .values(**values)- )- await session.execute(stmt)- await session.commit()-- # Reload via normal accessor so we return a fully-attached object- file_orm = await FileMetadataModel.read_async(- db_session=session,- identifier=file_id,- actor=actor,- )- return await file_orm.to_pydantic_async()-- @enforce_types- @trace_method- async def upsert_file_content(- self,- *,- file_id: str,- text: str,- actor: PydanticUser,- ) -> PydanticFileMetadata:- async with db_registry.async_session() as session:- await FileMetadataModel.read_async(session, file_id, actor)-- dialect_name = session.bind.dialect.name-- if dialect_name == "postgresql":- stmt = (- pg_insert(FileContentModel)- .values(file_id=file_id, text=text)- .on_conflict_do_update(- index_elements=[FileContentModel.file_id],- set_={"text": text},- )- )- await session.execute(stmt)- else:- # Emulate upsert for SQLite and others- stmt = select(FileContentModel).where(FileContentModel.file_id == file_id)- result = await session.execute(stmt)- existing = result.scalar_one_or_none()-- if existing:- await session.execute(update(FileContentModel).where(FileContentModel.file_id == file_id).values(text=text))- else:- session.add(FileContentModel(file_id=file_id, text=text))-- await session.commit()-- # Reload with content- query = select(FileMetadataModel).options(selectinload(FileMetadataModel.content)).where(FileMetadataModel.id == file_id)- result = await session.execute(query)- return await result.scalar_one().to_pydantic_async(include_content=True)-- @enforce_types- @trace_method- async def list_files(- self, source_id: str, actor: PydanticUser, after: Optional[str] = None, limit: Optional[int] = 50, include_content: bool = False- ) -> List[PydanticFileMetadata]:- """List all files with optional pagination."""- async with db_registry.async_session() as session:- options = [selectinload(FileMetadataModel.content)] if include_content else None-- files = await FileMetadataModel.list_async(- db_session=session,- after=after,- limit=limit,- organization_id=actor.organization_id,- source_id=source_id,- query_options=options,- )- return [await file.to_pydantic_async(include_content=include_content) for file in files]-- @enforce_types- @trace_method- async def delete_file(self, file_id: str, actor: PydanticUser) -> PydanticFileMetadata:- """Delete a file by its ID."""- async with db_registry.async_session() as session:- file = await FileMetadataModel.read_async(db_session=session, identifier=file_id)- await file.hard_delete_async(db_session=session, actor=actor)- return await file.to_pydantic_async()diff --git a/letta/services/tool_executor/files_tool_executor.py b/letta/services/tool_executor/files_tool_executor.pyindex b85fe6abc..ceed535fa 100644--- a/letta/services/tool_executor/files_tool_executor.py+++ b/letta/services/tool_executor/files_tool_executor.py@@ -10,6 +10,7 @@ from letta.schemas.tool_execution_result import ToolExecutionResultfrom letta.schemas.user import Userfrom letta.services.agent_manager import AgentManagerfrom letta.services.block_manager import BlockManager+from letta.services.file_manager import FileManagerfrom letta.services.file_processor.chunker.line_chunker import LineChunkerfrom letta.services.files_agents_manager import FileAgentManagerfrom letta.services.message_manager import MessageManager@@ -49,6 +50,7 @@ class LettaFileToolExecutor(ToolExecutor):# TODO: This should be passed in to for testing purposesself.files_agents_manager = FileAgentManager()+ self.file_manager = FileManager()self.source_manager = SourceManager()self.logger = get_logger(__name__)@@ -112,7 +114,7 @@ class LettaFileToolExecutor(ToolExecutor):)file_id = file_agent.file_id- file = await self.source_manager.get_file_by_id(file_id=file_id, actor=self.actor, include_content=True)+ file = await self.file_manager.get_file_by_id(file_id=file_id, actor=self.actor, include_content=True)# TODO: Inefficient, maybe we can pre-compute this# TODO: This is also not the best way to split things - would be cool to have "content aware" splitting@@ -234,7 +236,7 @@ class LettaFileToolExecutor(ToolExecutor):for file_agent in file_agents:# Load file content- file = await self.source_manager.get_file_by_id(file_id=file_agent.file_id, actor=self.actor, include_content=True)+ file = await self.file_manager.get_file_by_id(file_id=file_agent.file_id, actor=self.actor, include_content=True)if not file or not file.content:files_skipped += 1diff --git a/tests/test_managers.py b/tests/test_managers.pyindex 1e17bceaf..0d12fddbe 100644--- a/tests/test_managers.py+++ b/tests/test_managers.py@@ -183,7 +183,7 @@ async def other_source(server: SyncServer, default_user):@pytest.fixtureasync def default_file(server: SyncServer, default_source, default_user, default_organization):- file = await server.source_manager.create_file(+ file = await server.file_manager.create_file(PydanticFileMetadata(file_name="test_file", organization_id=default_organization.id, source_id=default_source.id),actor=default_user,)@@ -681,7 +681,7 @@ async def another_file(server, default_source, default_user, default_organizatioorganization_id=default_organization.id,source_id=default_source.id,)- return await server.source_manager.create_file(pf, actor=default_user)+ return await server.file_manager.create_file(pf, actor=default_user)# ======================================================================================================================@@ -4581,10 +4581,10 @@ async def test_get_file_by_id(server: SyncServer, default_user, default_source):file_size=2048,source_id=default_source.id,)- created_file = await server.source_manager.create_file(file_metadata=file_metadata, actor=default_user)+ created_file = await server.file_manager.create_file(file_metadata=file_metadata, actor=default_user)# Retrieve the file by ID- retrieved_file = await server.source_manager.get_file_by_id(file_id=created_file.id, actor=default_user)+ retrieved_file = await server.file_manager.get_file_by_id(file_id=created_file.id, actor=default_user)# Assertions to verify the retrieved file matches the created oneassert retrieved_file.id == created_file.id@@ -4605,7 +4605,7 @@ async def test_create_and_retrieve_file_with_content(server, default_user, defausource_id=default_source.id,)- created = await server.source_manager.create_file(+ created = await server.file_manager.create_file(file_metadata=meta,actor=default_user,text=text_body,@@ -4618,7 +4618,7 @@ async def test_create_and_retrieve_file_with_content(server, default_user, defauassert await _count_file_content_rows(async_session, created.id) == 1# -- now fetch WITH the body- loaded = await server.source_manager.get_file_by_id(created.id, actor=default_user, include_content=True)+ loaded = await server.file_manager.get_file_by_id(created.id, actor=default_user, include_content=True)assert loaded.content == text_body@@ -4631,13 +4631,13 @@ async def test_create_file_without_content(server, default_user, default_source,file_size=123,source_id=default_source.id,)- created = await server.source_manager.create_file(file_metadata=meta, actor=default_user)+ created = await server.file_manager.create_file(file_metadata=meta, actor=default_user)# no content rowassert await _count_file_content_rows(async_session, created.id) == 0# include_content=True still works, returns None- loaded = await server.source_manager.get_file_by_id(created.id, actor=default_user, include_content=True)+ loaded = await server.file_manager.get_file_by_id(created.id, actor=default_user, include_content=True)assert loaded.content is None@@ -4652,7 +4652,7 @@ async def test_lazy_raise_guard(server, default_user, default_source, async_sessfile_size=len(text_body),source_id=default_source.id,)- created = await server.source_manager.create_file(file_metadata=meta, actor=default_user, text=text_body)+ created = await server.file_manager.create_file(file_metadata=meta, actor=default_user, text=text_body)# Grab ORM instance WITHOUT selectinload(FileMetadata.content)orm = await async_session.get(FileMetadataModel, created.id)@@ -4664,7 +4664,7 @@ async def test_lazy_raise_guard(server, default_user, default_source, async_sess@pytest.mark.asyncioasync def test_list_files_content_none(server, default_user, default_source):- files = await server.source_manager.list_files(source_id=default_source.id, actor=default_user)+ files = await server.file_manager.list_files(source_id=default_source.id, actor=default_user)assert all(f.content is None for f in files)@@ -4678,13 +4678,13 @@ async def test_delete_cascades_to_content(server, default_user, default_source,file_size=len(text_body),source_id=default_source.id,)- created = await server.source_manager.create_file(file_metadata=meta, actor=default_user, text=text_body)+ created = await server.file_manager.create_file(file_metadata=meta, actor=default_user, text=text_body)# ensure row exists firstassert await _count_file_content_rows(async_session, created.id) == 1# delete- await server.source_manager.delete_file(created.id, actor=default_user)+ await server.file_manager.delete_file(created.id, actor=default_user)# content row goneassert await _count_file_content_rows(async_session, created.id) == 0@@ -4694,29 +4694,27 @@ async def test_delete_cascades_to_content(server, default_user, default_source,async def test_list_files(server: SyncServer, default_user, default_source):"""Test listing files with pagination."""# Create multiple files- await server.source_manager.create_file(+ await server.file_manager.create_file(PydanticFileMetadata(file_name="File 1", file_path="/path/to/file1.txt", file_type="text/plain", source_id=default_source.id),actor=default_user,)if USING_SQLITE:time.sleep(CREATE_DELAY_SQLITE)- await server.source_manager.create_file(+ await server.file_manager.create_file(PydanticFileMetadata(file_name="File 2", file_path="/path/to/file2.txt", file_type="text/plain", source_id=default_source.id),actor=default_user,)# List files without pagination- files = await server.source_manager.list_files(source_id=default_source.id, actor=default_user)+ files = await server.file_manager.list_files(source_id=default_source.id, actor=default_user)assert len(files) == 2# List files with pagination- paginated_files = await server.source_manager.list_files(source_id=default_source.id, actor=default_user, limit=1)+ paginated_files = await server.file_manager.list_files(source_id=default_source.id, actor=default_user, limit=1)assert len(paginated_files) == 1# Ensure cursor-based pagination works- next_page = await server.source_manager.list_files(- source_id=default_source.id, actor=default_user, after=paginated_files[-1].id, limit=1- )+ next_page = await server.file_manager.list_files(source_id=default_source.id, actor=default_user, after=paginated_files[-1].id, limit=1)assert len(next_page) == 1assert next_page[0].file_name != paginated_files[0].file_name@@ -4727,16 +4725,16 @@ async def test_delete_file(server: SyncServer, default_user, default_source):file_metadata = PydanticFileMetadata(file_name="Delete File", file_path="/path/to/delete_file.txt", file_type="text/plain", source_id=default_source.id)- created_file = await server.source_manager.create_file(file_metadata=file_metadata, actor=default_user)+ created_file = await server.file_manager.create_file(file_metadata=file_metadata, actor=default_user)# Delete the file- deleted_file = await server.source_manager.delete_file(file_id=created_file.id, actor=default_user)+ deleted_file = await server.file_manager.delete_file(file_id=created_file.id, actor=default_user)# Assertions to verify deletionassert deleted_file.id == created_file.id# Verify that the file no longer appears in list_files- files = await server.source_manager.list_files(source_id=default_source.id, actor=default_user)+ files = await server.file_manager.list_files(source_id=default_source.id, actor=default_user)assert len(files) == 0@@ -4750,10 +4748,10 @@ async def test_update_file_status_basic(server, default_user, default_source):file_size=100,source_id=default_source.id,)- created = await server.source_manager.create_file(file_metadata=meta, actor=default_user)+ created = await server.file_manager.create_file(file_metadata=meta, actor=default_user)# Update status only- updated = await server.source_manager.update_file_status(+ updated = await server.file_manager.update_file_status(file_id=created.id,actor=default_user,processing_status=FileProcessingStatus.PARSING,@@ -4762,7 +4760,7 @@ async def test_update_file_status_basic(server, default_user, default_source):assert updated.error_message is None# Update both status and error message- updated = await server.source_manager.update_file_status(+ updated = await server.file_manager.update_file_status(file_id=created.id,actor=default_user,processing_status=FileProcessingStatus.ERROR,@@ -4782,9 +4780,9 @@ async def test_update_file_status_error_only(server, default_user, default_sourcfile_size=123,source_id=default_source.id,)- created = await server.source_manager.create_file(file_metadata=meta, actor=default_user)+ created = await server.file_manager.create_file(file_metadata=meta, actor=default_user)- updated = await server.source_manager.update_file_status(+ updated = await server.file_manager.update_file_status(file_id=created.id,actor=default_user,error_message="Timeout while embedding",@@ -4807,11 +4805,11 @@ async def test_upsert_file_content_basic(server: SyncServer, default_user, defaufile_size=len(initial_text),source_id=default_source.id,)- created = await server.source_manager.create_file(file_metadata=meta, actor=default_user)+ created = await server.file_manager.create_file(file_metadata=meta, actor=default_user)assert created.content is None# Step 2: Insert new content- file_with_content = await server.source_manager.upsert_file_content(+ file_with_content = await server.file_manager.upsert_file_content(file_id=created.id,text=initial_text,actor=default_user,@@ -4823,7 +4821,7 @@ async def test_upsert_file_content_basic(server: SyncServer, default_user, defauassert count == 1# Step 3: Update existing content- file_with_updated_content = await server.source_manager.upsert_file_content(+ file_with_updated_content = await server.file_manager.upsert_file_content(file_id=created.id,text=updated_text,actor=default_user,diff --git a/tests/test_sources.py b/tests/test_sources.pyindex b95e289b7..959863598 100644--- a/tests/test_sources.py+++ b/tests/test_sources.py@@ -58,6 +58,7 @@ def agent_state(client: LettaSDKClient):open_file_tool = client.tools.list(name="open_file")[0]close_file_tool = client.tools.list(name="close_file")[0]search_files_tool = client.tools.list(name="search_files")[0]+ grep_tool = client.tools.list(name="grep")[0]agent_state = client.agents.create(memory_blocks=[@@ -68,7 +69,7 @@ def agent_state(client: LettaSDKClient):],model="openai/gpt-4o-mini",embedding="openai/text-embedding-ada-002",- tool_ids=[open_file_tool.id, close_file_tool.id, search_files_tool.id],+ tool_ids=[open_file_tool.id, close_file_tool.id, search_files_tool.id, grep_tool.id],)yield agent_state