Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: Store entire parsed file content in separate table (#2618)
Matthew Zhoufb0dbbdf0dd2
diff --git a/alembic/versions/cdd4a1c11aee_add_file_name_to_fileagent_association_.py b/alembic/versions/cdd4a1c11aee_add_file_name_to_fileagent_association_.pynew file mode 100644index 000000000..164803d50--- /dev/null+++ b/alembic/versions/cdd4a1c11aee_add_file_name_to_fileagent_association_.py@@ -0,0 +1,63 @@+"""Add file_name to FileAgent association table and FileContent table++Revision ID: cdd4a1c11aee+Revises: 614c4e53b66e+Create Date: 2025-06-03 15:35:59.623704++"""++from typing import Sequence, Union++import sqlalchemy as sa++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "cdd4a1c11aee"+down_revision: Union[str, None] = "614c4e53b66e"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.create_table(+ "file_contents",+ sa.Column("file_id", sa.String(), nullable=False),+ sa.Column("text", sa.Text(), nullable=False),+ sa.Column("id", sa.String(), nullable=False),+ sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=True),+ sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=True),+ sa.Column("is_deleted", sa.Boolean(), server_default=sa.text("FALSE"), nullable=False),+ sa.Column("_created_by_id", sa.String(), nullable=True),+ sa.Column("_last_updated_by_id", sa.String(), nullable=True),+ sa.ForeignKeyConstraint(["file_id"], ["files.id"], ondelete="CASCADE"),+ sa.PrimaryKeyConstraint("file_id", "id"),+ )+ # add the column, nullable for now+ op.add_column("files_agents", sa.Column("file_name", sa.String(), nullable=True))++ # back-fill using a single UPDATE … FROM join+ op.execute(+ """+ UPDATE files_agents fa+ SET file_name = f.file_name+ FROM files f+ WHERE fa.file_id = f.id;+ """+ )++ # now make it NOT NULL+ op.alter_column("files_agents", "file_name", nullable=False)+ op.create_index("ix_files_agents_agent_file_name", "files_agents", ["agent_id", "file_name"], unique=False)+ op.create_unique_constraint("uq_files_agents_agent_file_name", "files_agents", ["agent_id", "file_name"])+ # ### end Alembic commands ###+++def downgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.drop_constraint("uq_files_agents_agent_file_name", "files_agents", type_="unique")+ op.drop_index("ix_files_agents_agent_file_name", table_name="files_agents")+ op.drop_column("files_agents", "file_name")+ op.drop_table("file_contents")+ # ### end Alembic commands ###diff --git a/letta/orm/file.py b/letta/orm/file.pyindex baf14d26d..017acc02f 100644--- a/letta/orm/file.py+++ b/letta/orm/file.py@@ -1,6 +1,8 @@+import uuidfrom typing import TYPE_CHECKING, List, Optional-from sqlalchemy import Integer, String+from sqlalchemy import ForeignKey, Integer, String, Text+from sqlalchemy.ext.asyncio import AsyncAttrsfrom sqlalchemy.orm import Mapped, mapped_column, relationshipfrom letta.orm.mixins import OrganizationMixin, SourceMixin@@ -14,7 +16,27 @@ if TYPE_CHECKING:from letta.orm.source import Source-class FileMetadata(SqlalchemyBase, OrganizationMixin, SourceMixin):+# TODO: Note that this is NOT organization scoped, this is potentially dangerous if we misuse this+# TODO: This should ONLY be manipulated internally in relation to FileMetadata.content+# TODO: Leaving organization_id out of this for now for simplicity+class FileContent(SqlalchemyBase):+ """Holds the full text content of a file (potentially large)."""++ __tablename__ = "file_contents"++ # TODO: We want to migrate all the ORM models to do this, so we will need to move this to the SqlalchemyBase+ # TODO: Some still rely on the Pydantic object to do this+ id: Mapped[str] = mapped_column(String, primary_key=True, default=lambda: f"file_content-{uuid.uuid4()}")+ file_id: Mapped[str] = mapped_column(+ ForeignKey("files.id", ondelete="CASCADE"), primary_key=True, doc="Foreign key to files table; also serves as primary key."+ )+ text: Mapped[str] = mapped_column(Text, nullable=False, doc="Full plain-text content of the file (e.g., extracted from a PDF).")++ # back-reference to FileMetadata+ file: Mapped["FileMetadata"] = relationship(back_populates="content", lazy="selectin")+++class FileMetadata(SqlalchemyBase, OrganizationMixin, SourceMixin, AsyncAttrs):"""Represents an uploaded file."""__tablename__ = "files"@@ -33,4 +55,46 @@ class FileMetadata(SqlalchemyBase, OrganizationMixin, SourceMixin):source_passages: Mapped[List["SourcePassage"]] = relationship("SourcePassage", back_populates="file", lazy="selectin", cascade="all, delete-orphan")- file_agents: Mapped[List["FileAgent"]] = relationship("FileAgent", back_populates="file", lazy="selectin")+ file_agents: Mapped[List["FileAgent"]] = relationship(+ "FileAgent",+ back_populates="file",+ lazy="selectin",+ cascade="all, delete-orphan",+ passive_deletes=True, # ← add this+ )+ content: Mapped[Optional["FileContent"]] = relationship(+ "FileContent",+ uselist=False,+ back_populates="file",+ lazy="raise", # raises if you access without eager load+ cascade="all, delete-orphan",+ )++ async def to_pydantic_async(self, include_content: bool = False) -> PydanticFileMetadata:+ """+ Async version of `to_pydantic` that supports optional relationship loading+ without requiring `expire_on_commit=False`.+ """++ # Load content relationship if requested+ if include_content:+ content_obj = await self.awaitable_attrs.content+ content_text = content_obj.text if content_obj else None+ else:+ content_text = None++ return PydanticFileMetadata(+ id=self.id,+ organization_id=self.organization_id,+ source_id=self.source_id,+ file_name=self.file_name,+ file_path=self.file_path,+ file_type=self.file_type,+ file_size=self.file_size,+ file_creation_date=self.file_creation_date,+ file_last_modified_date=self.file_last_modified_date,+ created_at=self.created_at,+ updated_at=self.updated_at,+ is_deleted=self.is_deleted,+ content=content_text,+ )diff --git a/letta/orm/files_agents.py b/letta/orm/files_agents.pyindex 19b25bca1..61131ce1a 100644--- a/letta/orm/files_agents.py+++ b/letta/orm/files_agents.py@@ -26,6 +26,8 @@ class FileAgent(SqlalchemyBase, OrganizationMixin):__table_args__ = (Index("ix_files_agents_file_id_agent_id", "file_id", "agent_id"),UniqueConstraint("file_id", "agent_id", name="uq_files_agents_file_agent"),+ UniqueConstraint("agent_id", "file_name", name="uq_files_agents_agent_file_name"),+ Index("ix_files_agents_agent_file_name", "agent_id", "file_name"),)__pydantic_model__ = PydanticFileAgent@@ -33,6 +35,7 @@ class FileAgent(SqlalchemyBase, OrganizationMixin):# TODO: Some still rely on the Pydantic object to do thisid: Mapped[str] = mapped_column(String, primary_key=True, default=lambda: f"file_agent-{uuid.uuid4()}")file_id: Mapped[str] = mapped_column(String, ForeignKey("files.id", ondelete="CASCADE"), primary_key=True, doc="ID of the file.")+ file_name: Mapped[str] = mapped_column(String, nullable=False, doc="Denormalized copy of files.file_name; unique per agent.")agent_id: Mapped[str] = mapped_column(String, ForeignKey("agents.id", ondelete="CASCADE"), primary_key=True, doc="ID of the agent.")is_open: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, doc="True if the agent currently has the file open.")@@ -55,6 +58,8 @@ class FileAgent(SqlalchemyBase, OrganizationMixin):"FileMetadata",foreign_keys=[file_id],lazy="selectin",+ back_populates="file_agents",+ passive_deletes=True, # ← add this)# TODO: This is temporary as we figure out if we want FileBlock as a first class citizendiff --git a/letta/orm/sqlalchemy_base.py b/letta/orm/sqlalchemy_base.pyindex 04e27e11b..ac0b6e46b 100644--- a/letta/orm/sqlalchemy_base.py+++ b/letta/orm/sqlalchemy_base.py@@ -4,10 +4,11 @@ from functools import wrapsfrom pprint import pformatfrom typing import TYPE_CHECKING, List, Literal, Optional, Tuple, Union-from sqlalchemy import String, and_, delete, func, or_, select, text+from sqlalchemy import Sequence, String, and_, delete, func, or_, select, textfrom sqlalchemy.exc import DBAPIError, IntegrityError, TimeoutErrorfrom sqlalchemy.ext.asyncio import AsyncSessionfrom sqlalchemy.orm import Mapped, Session, mapped_column+from sqlalchemy.orm.interfaces import ORMOptionfrom letta.log import get_loggerfrom letta.orm.base import Base, CommonSqlalchemyMetaMixins@@ -163,6 +164,7 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):join_conditions: Optional[Union[Tuple, List]] = None,identifier_keys: Optional[List[str]] = None,identity_id: Optional[str] = None,+ query_options: Sequence[ORMOption] | None = None, # ← new**kwargs,) -> List["SqlalchemyBase"]:"""@@ -224,6 +226,9 @@ class SqlalchemyBase(CommonSqlalchemyMetaMixins, Base):identity_id=identity_id,**kwargs,)+ if query_options:+ for opt in query_options:+ query = query.options(opt)# Execute the queryresults = await db_session.execute(query)diff --git a/letta/schemas/file.py b/letta/schemas/file.pyindex ecd9fe0a6..2e38b5993 100644--- a/letta/schemas/file.py+++ b/letta/schemas/file.py@@ -40,6 +40,11 @@ class FileMetadata(FileMetadataBase):updated_at: Optional[datetime] = Field(default_factory=datetime.utcnow, description="The update date of the file.")is_deleted: bool = Field(False, description="Whether this file is deleted or not.")+ # This is optional, and only occasionally pulled in since it can be very large+ content: Optional[str] = Field(+ default=None, description="Optional full-text content of the file; only populated on demand due to its size."+ )+class FileAgentBase(LettaBase):"""Base class for the FileMetadata-⇄-Agent association schemas"""diff --git a/letta/server/db.py b/letta/server/db.pyindex 5816b6dca..f50b09880 100644--- a/letta/server/db.py+++ b/letta/server/db.py@@ -131,7 +131,12 @@ class DatabaseRegistry:# Create async session factoryself._async_engines["default"] = async_engineself._async_session_factories["default"] = async_sessionmaker(- close_resets_only=False, autocommit=False, autoflush=False, bind=self._async_engines["default"], class_=AsyncSession+ expire_on_commit=True,+ close_resets_only=False,+ autocommit=False,+ autoflush=False,+ bind=self._async_engines["default"],+ class_=AsyncSession,)self._initialized["async"] = True@@ -207,11 +212,6 @@ class DatabaseRegistry:self.initialize_sync()return self._engines.get(name)- def get_async_engine(self, name: str = "default") -> AsyncEngine:- """Get an async database engine by name."""- self.initialize_async()- return self._async_engines.get(name)-def get_session_factory(self, name: str = "default") -> sessionmaker:"""Get a session factory by name."""self.initialize_sync()diff --git a/letta/server/rest_api/routers/v1/agents.py b/letta/server/rest_api/routers/v1/agents.pyindex b385a2a57..f416749f8 100644--- a/letta/server/rest_api/routers/v1/agents.py+++ b/letta/server/rest_api/routers/v1/agents.py@@ -12,7 +12,7 @@ from sqlalchemy.exc import IntegrityError, OperationalErrorfrom starlette.responses import Response, StreamingResponsefrom letta.agents.letta_agent import LettaAgent-from letta.constants import CORE_MEMORY_SOURCE_CHAR_LIMIT, DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG+from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARGfrom letta.groups.sleeptime_multi_agent_v2 import SleeptimeMultiAgentV2from letta.helpers.datetime_helpers import get_utc_timestamp_nsfrom letta.log import get_logger@@ -311,20 +311,18 @@ async def attach_source(actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)agent_state = await server.agent_manager.attach_source_async(agent_id=agent_id, source_id=source_id, actor=actor)- files = await server.source_manager.list_files(source_id, actor)+ files = await server.source_manager.list_files(source_id, actor, include_content=True)texts = []file_ids = []+ file_names = []for f in files:- passages = await server.passage_manager.list_passages_by_file_id_async(file_id=f.id, actor=actor)- passage_text = ""- for p in passages:- if len(passage_text) <= CORE_MEMORY_SOURCE_CHAR_LIMIT:- passage_text += p.text-- texts.append(passage_text)+ texts.append(f.content if f.content else "")file_ids.append(f.id)+ file_names.append(f.file_name)- await server.insert_files_into_context_window(agent_state=agent_state, texts=texts, file_ids=file_ids, actor=actor)+ await server.insert_files_into_context_window(+ agent_state=agent_state, texts=texts, file_ids=file_ids, file_names=file_names, actor=actor+ )if agent_state.enable_sleeptime:source = await server.source_manager.get_source_by_id(source_id=source_id)diff --git a/letta/server/rest_api/routers/v1/sources.py b/letta/server/rest_api/routers/v1/sources.pyindex cc57d9773..9c72258a4 100644--- a/letta/server/rest_api/routers/v1/sources.py+++ b/letta/server/rest_api/routers/v1/sources.py@@ -270,14 +270,21 @@ async def list_source_files(source_id: str,limit: int = Query(1000, description="Number of files to return"),after: Optional[str] = Query(None, description="Pagination cursor to fetch the next set of results"),+ include_content: bool = Query(False, description="Whether to include full file content"),server: "SyncServer" = Depends(get_letta_server),- actor_id: Optional[str] = Header(None, alias="user_id"), # Extract user_id from header, default to None if not present+ actor_id: Optional[str] = Header(None, alias="user_id"),):"""List paginated files associated with a data source."""actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)- return await server.source_manager.list_files(source_id=source_id, limit=limit, after=after, actor=actor)+ return await server.source_manager.list_files(+ source_id=source_id,+ limit=limit,+ after=after,+ actor=actor,+ include_content=include_content,+ )# it's redundant to include /delete in the URL path. The HTTP verb DELETE already implies that action.diff --git a/letta/server/server.py b/letta/server/server.pyindex b83a70ce2..8e79096e0 100644--- a/letta/server/server.py+++ b/letta/server/server.py@@ -1368,12 +1368,14 @@ class SyncServer(Server):)await self.agent_manager.delete_agent_async(agent_id=sleeptime_agent_state.id, actor=actor)- async def _upsert_file_to_agent(self, agent_id: str, text: str, file_id: str, actor: User) -> None:+ async def _upsert_file_to_agent(self, agent_id: str, text: str, file_id: str, file_name: str, actor: User) -> None:"""Internal method to create or update a file <-> agent association"""truncated_text = text[:CORE_MEMORY_SOURCE_CHAR_LIMIT]- await self.file_agent_manager.attach_file(agent_id=agent_id, file_id=file_id, actor=actor, visible_content=truncated_text)+ await self.file_agent_manager.attach_file(+ agent_id=agent_id, file_id=file_id, file_name=file_name, actor=actor, visible_content=truncated_text+ )async def _remove_file_from_agent(self, agent_id: str, file_id: str, actor: User) -> None:"""@@ -1389,7 +1391,7 @@ class SyncServer(Server):logger.info(f"File {file_id} already removed from agent {agent_id}, skipping...")async def insert_file_into_context_windows(- self, source_id: str, text: str, file_id: str, actor: User, agent_states: Optional[List[AgentState]] = None+ self, source_id: str, text: str, file_id: str, file_name: str, actor: User, agent_states: Optional[List[AgentState]] = None) -> List[AgentState]:"""Insert the uploaded document into the context window of all agents@@ -1404,11 +1406,13 @@ class SyncServer(Server):logger.info(f"Inserting document into context window for source: {source_id}")logger.info(f"Attached agents: {[a.id for a in agent_states]}")- await asyncio.gather(*(self._upsert_file_to_agent(agent_state.id, text, file_id, actor) for agent_state in agent_states))+ await asyncio.gather(*(self._upsert_file_to_agent(agent_state.id, text, file_id, file_name, actor) for agent_state in agent_states))return agent_states- async def insert_files_into_context_window(self, agent_state: AgentState, texts: List[str], file_ids: List[str], actor: User) -> None:+ async def insert_files_into_context_window(+ self, agent_state: AgentState, texts: List[str], file_ids: List[str], file_names: List[str], actor: User+ ) -> None:"""Insert the uploaded documents into the context window of an agentattached to the given source.@@ -1418,7 +1422,12 @@ class SyncServer(Server):if len(texts) != len(file_ids):raise ValueError(f"Mismatch between number of texts ({len(texts)}) and file ids ({len(file_ids)})")- await asyncio.gather(*(self._upsert_file_to_agent(agent_state.id, text, file_id, actor) for text, file_id in zip(texts, file_ids)))+ await asyncio.gather(+ *(+ self._upsert_file_to_agent(agent_state.id, text, file_id, file_name, actor)+ for text, file_id, file_name in zip(texts, file_ids, file_names)+ )+ )async def remove_file_from_context_windows(self, source_id: str, file_id: str, actor: User) -> None:"""diff --git a/letta/services/file_processor/file_processor.py b/letta/services/file_processor/file_processor.pyindex 0854d049a..a580ff7ab 100644--- a/letta/services/file_processor/file_processor.py+++ b/letta/services/file_processor/file_processor.py@@ -52,7 +52,6 @@ class FileProcessor:job: Optional[Job] = None,) -> List[Passage]:file_metadata = self._extract_upload_file_metadata(file, source_id=source_id)- file_metadata = await self.source_manager.create_file(file_metadata, self.actor)filename = file_metadata.file_nametry:@@ -66,11 +65,16 @@ class FileProcessor:logger.info(f"Starting OCR extraction for {filename}")ocr_response = await self.file_parser.extract_text(content, mime_type=file_metadata.file_type)+ # persist file with raw text+ raw_markdown_text = "".join([page.markdown for page in ocr_response.pages])+ await self.source_manager.create_file(file_metadata, self.actor, text=raw_markdown_text)+if not ocr_response or len(ocr_response.pages) == 0:raise ValueError("No text extracted from PDF")logger.info("Chunking extracted text")all_passages = []+for page in ocr_response.pages:chunks = self.text_chunker.chunk_text(page)@@ -88,8 +92,9 @@ class FileProcessor:await server.insert_file_into_context_windows(source_id=source_id,- text="".join([ocr_response.pages[i].markdown for i in range(min(3, len(ocr_response.pages)))]),+ text=raw_markdown_text,file_id=file_metadata.id,+ file_name=file_metadata.file_name,actor=self.actor,agent_states=agent_states,)diff --git a/letta/services/files_agents_manager.py b/letta/services/files_agents_manager.pyindex 155e83221..1d4d0071a 100644--- a/letta/services/files_agents_manager.py+++ b/letta/services/files_agents_manager.py@@ -22,6 +22,7 @@ class FileAgentManager:*,agent_id: str,file_id: str,+ file_name: str,actor: PydanticUser,is_open: bool = True,visible_content: Optional[str] = None,@@ -38,6 +39,7 @@ class FileAgentManager:and_(FileAgentModel.agent_id == agent_id,FileAgentModel.file_id == file_id,+ FileAgentModel.file_name == file_name,FileAgentModel.organization_id == actor.organization_id,))@@ -61,6 +63,7 @@ class FileAgentManager:assoc = FileAgentModel(agent_id=agent_id,file_id=file_id,+ file_name=file_name,organization_id=actor.organization_id,is_open=is_open,visible_content=visible_content,diff --git a/letta/services/source_manager.py b/letta/services/source_manager.pyindex 7580f77e4..099441b5c 100644--- a/letta/services/source_manager.py+++ b/letta/services/source_manager.py@@ -1,9 +1,15 @@import asynciofrom typing import List, Optional+from sqlalchemy import select+from sqlalchemy.exc import IntegrityError+from sqlalchemy.orm import selectinload+from letta.orm.errors import NoResultFound+from letta.orm.file import FileContent as FileContentModelfrom letta.orm.file import FileMetadata as FileMetadataModelfrom letta.orm.source import Source as SourceModel+from letta.orm.sqlalchemy_base import AccessTypefrom letta.schemas.agent import AgentState as PydanticAgentStatefrom letta.schemas.file import FileMetadata as PydanticFileMetadatafrom letta.schemas.source import Source as PydanticSource@@ -142,41 +148,102 @@ class SourceManager:@enforce_types@trace_method- async def create_file(self, file_metadata: PydanticFileMetadata, actor: PydanticUser) -> PydanticFileMetadata:- """Create a new file based on the PydanticFileMetadata schema."""- db_file = await self.get_file_by_id(file_metadata.id, actor=actor)- if db_file:- return db_file- else:- async with db_registry.async_session() as session:+ async def create_file(+ self,+ file_metadata: PydanticFileMetadata,+ actor: PydanticUser,+ *,+ text: Optional[str] = None,+ ) -> PydanticFileMetadata:++ # short-circuit if it already exists+ existing = await self.get_file_by_id(file_metadata.id, actor=actor)+ if existing:+ return existing++ async with db_registry.async_session() as session:+ try:file_metadata.organization_id = actor.organization_id- file_metadata = FileMetadataModel(**file_metadata.model_dump(to_orm=True, exclude_none=True))- await file_metadata.create_async(session, actor=actor)- return file_metadata.to_pydantic()+ file_orm = FileMetadataModel(**file_metadata.model_dump(to_orm=True, exclude_none=True))+ await file_orm.create_async(session, actor=actor, no_commit=True)++ if text is not None:+ content_orm = FileContentModel(file_id=file_orm.id, text=text)+ await content_orm.create_async(session, actor=actor, no_commit=True)++ await session.commit()+ await session.refresh(file_orm)+ return await file_orm.to_pydantic_async()++ except IntegrityError:+ await session.rollback()+ return await self.get_file_by_id(file_metadata.id, actor=actor)# TODO: We make actor optional for now, but should most likely be enforced due to security reasons@enforce_types@trace_method- async def get_file_by_id(self, file_id: str, actor: Optional[PydanticUser] = None) -> Optional[PydanticFileMetadata]:- """Retrieve a file by its ID."""+ async def get_file_by_id(+ self,+ file_id: str,+ actor: Optional[PydanticUser] = None,+ *,+ include_content: bool = False,+ ) -> Optional[PydanticFileMetadata]:+ """Retrieve a file by its ID.++ If `include_content=True`, the FileContent relationship is eagerly+ loaded so `to_pydantic(include_content=True)` never triggers a+ lazy SELECT (avoids MissingGreenlet).+ """async with db_registry.async_session() as session:try:- file = await FileMetadataModel.read_async(db_session=session, identifier=file_id, actor=actor)- return file.to_pydantic()+ if include_content:+ # explicit eager load+ query = (+ select(FileMetadataModel).where(FileMetadataModel.id == file_id).options(selectinload(FileMetadataModel.content))+ )+ # apply org-scoping if actor provided+ if actor:+ query = FileMetadataModel.apply_access_predicate(+ query,+ actor,+ access=["read"],+ access_type=AccessType.ORGANIZATION,+ )++ result = await session.execute(query)+ file_orm = result.scalar_one()+ else:+ # fast path (metadata only)+ file_orm = await FileMetadataModel.read_async(+ db_session=session,+ identifier=file_id,+ actor=actor,+ )++ return await file_orm.to_pydantic_async(include_content=include_content)+except NoResultFound:return None@enforce_types@trace_methodasync def list_files(- self, source_id: str, actor: PydanticUser, after: Optional[str] = None, limit: Optional[int] = 50+ self, source_id: str, actor: PydanticUser, after: Optional[str] = None, limit: Optional[int] = 50, include_content: bool = False) -> List[PydanticFileMetadata]:"""List all files with optional pagination."""async with db_registry.async_session() as session:+ options = [selectinload(FileMetadataModel.content)] if include_content else None+files = await FileMetadataModel.list_async(- db_session=session, after=after, limit=limit, organization_id=actor.organization_id, source_id=source_id+ db_session=session,+ after=after,+ limit=limit,+ organization_id=actor.organization_id,+ source_id=source_id,+ query_options=options,)- return [file.to_pydantic() for file in files]+ return [await file.to_pydantic_async(include_content=include_content) for file in files]@enforce_types@trace_method@@ -185,4 +252,4 @@ class SourceManager:async with db_registry.async_session() as session:file = await FileMetadataModel.read_async(db_session=session, identifier=file_id)await file.hard_delete_async(db_session=session, actor=actor)- return file.to_pydantic()+ return await file.to_pydantic_async()diff --git a/tests/test_managers.py b/tests/test_managers.pyindex f8a42d824..007f59938 100644--- a/tests/test_managers.py+++ b/tests/test_managers.py@@ -9,12 +9,15 @@ from datetime import datetime, timedelta, timezonefrom typing import Listimport httpx++# tests/test_file_content_flow.pyimport pytestfrom anthropic.types.beta import BetaMessagefrom anthropic.types.beta.messages import BetaMessageBatchIndividualResponse, BetaMessageBatchSucceededResultfrom openai.types.chat.chat_completion_message_tool_call import ChatCompletionMessageToolCall as OpenAIToolCallfrom openai.types.chat.chat_completion_message_tool_call import Function as OpenAIFunction-from sqlalchemy.exc import IntegrityError+from sqlalchemy import func, select+from sqlalchemy.exc import IntegrityError, InvalidRequestErrorfrom sqlalchemy.orm.exc import StaleDataErrorfrom letta.config import LettaConfig@@ -39,6 +42,8 @@ from letta.orm import Base, Blockfrom letta.orm.block_history import BlockHistoryfrom letta.orm.enums import ActorType, JobType, ToolTypefrom letta.orm.errors import NoResultFound, UniqueConstraintViolationError+from letta.orm.file import FileContent as FileContentModel+from letta.orm.file import FileMetadata as FileMetadataModelfrom letta.schemas.agent import AgentStepState, CreateAgent, UpdateAgentfrom letta.schemas.block import Block as PydanticBlockfrom letta.schemas.block import BlockUpdate, CreateBlock@@ -93,15 +98,26 @@ CREATE_DELAY_SQLITE = 1USING_SQLITE = not bool(os.getenv("LETTA_PG_URI"))[email protected](autouse=True)-async def _clear_tables():+async def _count_file_content_rows(session, file_id: str) -> int:+ q = select(func.count()).select_from(FileContentModel).where(FileContentModel.file_id == file_id)+ result = await session.execute(q)+ return result.scalar_one()+++async def async_session():async with db_registry.async_session() as session:- for table in reversed(Base.metadata.sorted_tables): # Reverse to avoid FK issues- # If this is the block_history table, skip it- if table.name == "block_history":- continue- await session.execute(table.delete()) # Truncate table- await session.commit()+ yield session++[email protected](autouse=True)+async def _clear_tables(async_session):+ for table in reversed(Base.metadata.sorted_tables): # Reverse to avoid FK issues+ # If this is the block_history table, skip it+ if table.name == "block_history":+ continue+ await async_session.execute(table.delete()) # Truncate table+ await async_session.commit()@pytest.fixture@@ -645,6 +661,7 @@ async def file_attachment(server, default_user, sarah_agent, default_file):assoc = await server.file_agent_manager.attach_file(agent_id=sarah_agent.id,file_id=default_file.id,+ file_name=default_file.file_name,actor=default_user,visible_content="initial",)@@ -699,6 +716,7 @@ async def test_get_context_window_basic(server: SyncServer, comprehensive_test_aassoc = await server.file_agent_manager.attach_file(agent_id=created_agent.id,file_id=default_file.id,+ file_name=default_file.file_name,actor=default_user,visible_content="hello",)@@ -4077,6 +4095,103 @@ async def test_get_file_by_id(server: SyncServer, default_user, default_source):assert retrieved_file.file_type == created_file.file_type+async def test_create_and_retrieve_file_with_content(server, default_user, default_source, async_session):+ text_body = "Line 1\nLine 2\nLine 3"++ meta = PydanticFileMetadata(+ file_name="with_body.txt",+ file_path="/tmp/with_body.txt",+ file_type="text/plain",+ file_size=len(text_body),+ source_id=default_source.id,+ )++ created = await server.source_manager.create_file(+ file_metadata=meta,+ actor=default_user,+ text=text_body,+ )++ # -- metadata-only return: content is NOT present+ assert created.content is None++ # body row exists+ assert await _count_file_content_rows(async_session, created.id) == 1++ # -- now fetch WITH the body+ loaded = await server.source_manager.get_file_by_id(created.id, actor=default_user, include_content=True)+ assert loaded.content == text_body+++async def test_create_file_without_content(server, default_user, default_source, async_session):+ meta = PydanticFileMetadata(+ file_name="no_body.txt",+ file_path="/tmp/no_body.txt",+ file_type="text/plain",+ file_size=123,+ source_id=default_source.id,+ )+ created = await server.source_manager.create_file(file_metadata=meta, actor=default_user)++ # no content row+ assert await _count_file_content_rows(async_session, created.id) == 0++ # include_content=True still works, returns None+ loaded = await server.source_manager.get_file_by_id(created.id, actor=default_user, include_content=True)+ assert loaded.content is None+++async def test_lazy_raise_guard(server, default_user, default_source, async_session):+ text_body = "lazy-raise"++ meta = PydanticFileMetadata(+ file_name="lazy_raise.txt",+ file_path="/tmp/lazy_raise.txt",+ file_type="text/plain",+ file_size=len(text_body),+ source_id=default_source.id,+ )+ created = await server.source_manager.create_file(file_metadata=meta, actor=default_user, text=text_body)++ # Grab ORM instance WITHOUT selectinload(FileMetadata.content)+ orm = await async_session.get(FileMetadataModel, created.id)++ # to_pydantic(include_content=True) should raise – guard works+ with pytest.raises(InvalidRequestError):+ await orm.to_pydantic_async(include_content=True)+++async def test_list_files_content_none(server, default_user, default_source):+ files = await server.source_manager.list_files(source_id=default_source.id, actor=default_user)+ assert all(f.content is None for f in files)+++async def test_delete_cascades_to_content(server, default_user, default_source, async_session):+ text_body = "to be deleted"+ meta = PydanticFileMetadata(+ file_name="delete_me.txt",+ file_path="/tmp/delete_me.txt",+ file_type="text/plain",+ file_size=len(text_body),+ source_id=default_source.id,+ )+ created = await server.source_manager.create_file(file_metadata=meta, actor=default_user, text=text_body)++ # ensure row exists first+ assert await _count_file_content_rows(async_session, created.id) == 1++ # delete+ await server.source_manager.delete_file(created.id, actor=default_user)++ # content row gone+ assert await _count_file_content_rows(async_session, created.id) == 0++@pytest.mark.asyncioasync def test_list_files(server: SyncServer, default_user, default_source):"""Test listing files with pagination."""@@ -5811,6 +5926,7 @@ async def test_attach_creates_association(server, default_user, sarah_agent, defassoc = await server.file_agent_manager.attach_file(agent_id=sarah_agent.id,file_id=default_file.id,+ file_name=default_file.file_name,actor=default_user,visible_content="hello",)@@ -5832,6 +5948,7 @@ async def test_attach_is_idempotent(server, default_user, sarah_agent, default_fa1 = await server.file_agent_manager.attach_file(agent_id=sarah_agent.id,file_id=default_file.id,+ file_name=default_file.file_name,actor=default_user,visible_content="first",)@@ -5840,6 +5957,7 @@ async def test_attach_is_idempotent(server, default_user, sarah_agent, default_fa2 = await server.file_agent_manager.attach_file(agent_id=sarah_agent.id,file_id=default_file.id,+ file_name=default_file.file_name,actor=default_user,is_open=False,visible_content="second",@@ -5900,11 +6018,17 @@ async def test_list_files_and_agents(another_file,):# default_file ↔ charles (open)- await server.file_agent_manager.attach_file(agent_id=charles_agent.id, file_id=default_file.id, actor=default_user)+ await server.file_agent_manager.attach_file(+ agent_id=charles_agent.id, file_id=default_file.id, file_name=default_file.file_name, actor=default_user+ )# default_file ↔ sarah (open)- await server.file_agent_manager.attach_file(agent_id=sarah_agent.id, file_id=default_file.id, actor=default_user)+ await server.file_agent_manager.attach_file(+ agent_id=sarah_agent.id, file_id=default_file.id, file_name=default_file.file_name, actor=default_user+ )# another_file ↔ sarah (closed)- await server.file_agent_manager.attach_file(agent_id=sarah_agent.id, file_id=another_file.id, actor=default_user, is_open=False)+ await server.file_agent_manager.attach_file(+ agent_id=sarah_agent.id, file_id=another_file.id, file_name=another_file.file_name, actor=default_user, is_open=False+ )files_for_sarah = await server.file_agent_manager.list_files_for_agent(sarah_agent.id, actor=default_user)assert {f.file_id for f in files_for_sarah} == {default_file.id, another_file.id}@@ -5952,6 +6076,7 @@ async def test_org_scoping(await server.file_agent_manager.attach_file(agent_id=sarah_agent.id,file_id=default_file.id,+ file_name=default_file.file_name,actor=default_user,)