Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: Support exporting/importing sources and files for agent file (#3414)
Matthew Zhoub0138ecad695
diff --git a/letta/orm/organization.py b/letta/orm/organization.pyindex f5f65cb92..4f78ec895 100644--- a/letta/orm/organization.py+++ b/letta/orm/organization.py@@ -6,18 +6,17 @@ from letta.orm.sqlalchemy_base import SqlalchemyBasefrom letta.schemas.organization import Organization as PydanticOrganizationif TYPE_CHECKING:+ from letta.orm import Sourcefrom letta.orm.agent import Agent- from letta.orm.agent_passage import AgentPassagefrom letta.orm.block import Blockfrom letta.orm.group import Groupfrom letta.orm.identity import Identity- from letta.orm.llm_batch_item import LLMBatchItem+ from letta.orm.llm_batch_items import LLMBatchItemfrom letta.orm.llm_batch_job import LLMBatchJobfrom letta.orm.message import Message+ from letta.orm.passage import AgentPassage, SourcePassagefrom letta.orm.provider import Provider- from letta.orm.sandbox_config import AgentEnvironmentVariable, SandboxConfig- from letta.orm.sandbox_environment_variable import SandboxEnvironmentVariable- from letta.orm.source_passage import SourcePassage+ from letta.orm.sandbox_config import AgentEnvironmentVariable, SandboxConfig, SandboxEnvironmentVariablefrom letta.orm.tool import Toolfrom letta.orm.user import User@@ -48,6 +47,7 @@ class Organization(SqlalchemyBase):# relationshipsagents: Mapped[List["Agent"]] = relationship("Agent", back_populates="organization", cascade="all, delete-orphan")+ sources: Mapped[List["Source"]] = relationship("Source", cascade="all, delete-orphan")messages: Mapped[List["Message"]] = relationship("Message", back_populates="organization", cascade="all, delete-orphan")source_passages: Mapped[List["SourcePassage"]] = relationship("SourcePassage", back_populates="organization", cascade="all, delete-orphan"diff --git a/letta/schemas/agent_file.py b/letta/schemas/agent_file.pyindex ae669b921..c9a18072f 100644--- a/letta/schemas/agent_file.py+++ b/letta/schemas/agent_file.py@@ -1,14 +1,15 @@+from datetime import datetimefrom typing import Dict, List, Optionalfrom pydantic import BaseModel, Fieldfrom letta.schemas.agent import AgentState, CreateAgent-from letta.schemas.block import CreateBlock+from letta.schemas.block import Block, CreateBlockfrom letta.schemas.enums import MessageRole-from letta.schemas.file import FileMetadataCreate+from letta.schemas.file import FileAgent, FileAgentBase, FileMetadata, FileMetadataBasefrom letta.schemas.group import GroupCreatefrom letta.schemas.message import Message, MessageCreate-from letta.schemas.source import SourceCreate+from letta.schemas.source import Source, SourceCreatefrom letta.schemas.tool import Toolfrom letta.schemas.user import Userfrom letta.services.message_manager import MessageManager@@ -63,6 +64,30 @@ class MessageSchema(MessageCreate):)+class FileAgentSchema(FileAgentBase):+ """File-Agent relationship with human-readable ID for agent file"""++ __id_prefix__ = "file_agent"+ id: str = Field(..., description="Human-readable identifier for this file-agent relationship in the file")++ @classmethod+ def from_file_agent(cls, file_agent: FileAgent) -> "FileAgentSchema":+ """Convert FileAgent to FileAgentSchema"""++ create_file_agent = FileAgentBase(+ agent_id=file_agent.agent_id,+ file_id=file_agent.file_id,+ source_id=file_agent.source_id,+ file_name=file_agent.file_name,+ is_open=file_agent.is_open,+ visible_content=file_agent.visible_content,+ last_accessed_at=file_agent.last_accessed_at,+ )++ # Create FileAgentSchema with the file_agent's ID (will be remapped later)+ return cls(id=file_agent.id, **create_file_agent.model_dump())++class AgentSchema(CreateAgent):"""Agent with human-readable ID for agent file"""@@ -72,9 +97,12 @@ class AgentSchema(CreateAgent):default_factory=list, description="List of message IDs that are currently in the agent's context")messages: List[MessageSchema] = Field(default_factory=list, description="List of messages in the agent's conversation history")+ files_agents: List[FileAgentSchema] = Field(default_factory=list, description="List of file-agent relationships for this agent")@classmethod- async def from_agent_state(cls, agent_state: AgentState, message_manager: MessageManager, actor: User) -> "AgentSchema":+ async def from_agent_state(+ cls, agent_state: AgentState, message_manager: MessageManager, files_agents: List[FileAgent], actor: User+ ) -> "AgentSchema":"""Convert AgentState to AgentSchema"""create_agent = CreateAgent(@@ -131,6 +159,7 @@ class AgentSchema(CreateAgent):id=agent_state.id,in_context_message_ids=agent_state.message_ids or [],messages=message_schemas, # Messages will be populated separately by the manager+ files_agents=[FileAgentSchema.from_file_agent(f) for f in files_agents],**create_agent.model_dump(),)@@ -149,7 +178,7 @@ class BlockSchema(CreateBlock):id: str = Field(..., description="Human-readable identifier for this block in the file")@classmethod- def from_block(cls, block) -> "BlockSchema":+ def from_block(cls, block: Block) -> "BlockSchema":"""Convert Block to BlockSchema"""create_block = CreateBlock(@@ -168,12 +197,35 @@ class BlockSchema(CreateBlock):return cls(id=block.id, **create_block.model_dump())-class FileSchema(FileMetadataCreate):+class FileSchema(FileMetadataBase):"""File with human-readable ID for agent file"""__id_prefix__ = "file"id: str = Field(..., description="Human-readable identifier for this file in the file")+ @classmethod+ def from_file_metadata(cls, file_metadata: FileMetadata) -> "FileSchema":+ """Convert FileMetadata to FileSchema"""++ create_file = FileMetadataBase(+ source_id=file_metadata.source_id,+ file_name=file_metadata.file_name,+ original_file_name=file_metadata.original_file_name,+ file_path=file_metadata.file_path,+ file_type=file_metadata.file_type,+ file_size=file_metadata.file_size,+ file_creation_date=file_metadata.file_creation_date,+ file_last_modified_date=file_metadata.file_last_modified_date,+ processing_status=file_metadata.processing_status,+ error_message=file_metadata.error_message,+ total_chunks=file_metadata.total_chunks,+ chunks_embedded=file_metadata.chunks_embedded,+ content=file_metadata.content,+ )++ # Create FileSchema with the file's ID (will be remapped later)+ return cls(id=file_metadata.id, **create_file.model_dump())+class SourceSchema(SourceCreate):"""Source with human-readable ID for agent file"""@@ -181,6 +233,21 @@ class SourceSchema(SourceCreate):__id_prefix__ = "source"id: str = Field(..., description="Human-readable identifier for this source in the file")+ @classmethod+ def from_source(cls, source: Source) -> "SourceSchema":+ """Convert Block to BlockSchema"""++ create_block = SourceCreate(+ name=source.name,+ description=source.description,+ instructions=source.instructions,+ metadata=source.metadata,+ embedding_config=source.embedding_config,+ )++ # Create SourceSchema with the block's ID (will be remapped later)+ return cls(id=source.id, **create_block.model_dump())+# TODO: This one is quite thin, just a wrapper over Toolclass ToolSchema(Tool):@@ -200,13 +267,6 @@ class ToolSchema(Tool):# id: str = Field(..., description="Human-readable identifier for this MCP server in the file")-# class FileAgentStateSchema(FileAgentCreate):-# """File-Agent relationship with human-readable ID for agent file"""-#-# __id_prefix__ = "file_agent"-# id: str = Field(..., description="Human-readable identifier for this file-agent relationship in the file")--class AgentFileSchema(BaseModel):"""Schema for serialized agent file that can be exported to JSON and imported into agent server."""@@ -220,3 +280,4 @@ class AgentFileSchema(BaseModel):metadata: Dict[str, str] = Field(default_factory=dict, description="Metadata for this agent file, including revision_id and other export information.")+ created_at: Optional[datetime] = Field(default=None, description="The timestamp when the object was created.")diff --git a/letta/schemas/file.py b/letta/schemas/file.pyindex b74da074b..4a3c876f8 100644--- a/letta/schemas/file.py+++ b/letta/schemas/file.py@@ -74,6 +74,10 @@ class FileAgentBase(LettaBase):None,description="Portion of the file the agent is focused on (may be large).",)+ last_accessed_at: Optional[datetime] = Field(+ default_factory=datetime.utcnow,+ description="UTC timestamp of the agent’s most recent access to this file.",+ )class FileAgent(FileAgentBase):@@ -94,10 +98,6 @@ class FileAgent(FileAgentBase):None,description="Org ID this association belongs to (inherited from both agent and file).",)- last_accessed_at: Optional[datetime] = Field(- default_factory=datetime.utcnow,- description="UTC timestamp of the agent’s most recent access to this file.",- )created_at: Optional[datetime] = Field(default_factory=datetime.utcnow,@@ -108,11 +108,3 @@ class FileAgent(FileAgentBase):description="Row last-update timestamp (UTC).",)is_deleted: bool = Field(False, description="Soft-delete flag.")---class FileMetadataCreate(FileMetadataBase):- """Schema for creating a new FileMetadata"""---class FileAgentCreate(FileAgentBase):- """Schema for creating a new FileAgent association"""diff --git a/letta/services/agent_file_manager.py b/letta/services/agent_file_manager.pyindex d58c3d418..001cd964d 100644--- a/letta/services/agent_file_manager.py+++ b/letta/services/agent_file_manager.py@@ -1,3 +1,4 @@+from datetime import datetime, timezonefrom typing import Dict, Listfrom letta.errors import AgentFileExportError, AgentFileImportError@@ -7,6 +8,7 @@ from letta.schemas.agent_file import (AgentFileSchema,AgentSchema,BlockSchema,+ FileAgentSchema,FileSchema,GroupSchema,ImportResult,@@ -15,12 +17,17 @@ from letta.schemas.agent_file import (ToolSchema,)from letta.schemas.block import Block+from letta.schemas.file import FileMetadatafrom letta.schemas.message import Message+from letta.schemas.source import Sourcefrom letta.schemas.tool import Toolfrom letta.schemas.user import Userfrom letta.services.agent_manager import AgentManagerfrom letta.services.block_manager import BlockManagerfrom letta.services.file_manager import FileManager+from letta.services.file_processor.embedder.base_embedder import BaseEmbedder+from letta.services.file_processor.file_processor import FileProcessor+from letta.services.file_processor.parser.mistral_parser import MistralFileParserfrom letta.services.files_agents_manager import FileAgentManagerfrom letta.services.group_manager import GroupManagerfrom letta.services.mcp_manager import MCPManager@@ -54,6 +61,9 @@ class AgentFileManager:file_manager: FileManager,file_agent_manager: FileAgentManager,message_manager: MessageManager,+ embedder: BaseEmbedder,+ file_parser: MistralFileParser,+ using_pinecone: bool = False,):self.agent_manager = agent_managerself.tool_manager = tool_manager@@ -64,6 +74,9 @@ class AgentFileManager:self.file_manager = file_managerself.file_agent_manager = file_agent_managerself.message_manager = message_manager+ self.embedder = embedder+ self.file_parser = file_parser+ self.using_pinecone = using_pinecone# ID mapping state for exportself._db_to_file_ids: Dict[str, str] = {}@@ -77,6 +90,7 @@ class AgentFileManager:SourceSchema.__id_prefix__: 0,ToolSchema.__id_prefix__: 0,MessageSchema.__id_prefix__: 0,+ FileAgentSchema.__id_prefix__: 0,# MCPServerSchema.__id_prefix__: 0,}@@ -134,45 +148,66 @@ class AgentFileManager:return sorted(unique_blocks.values(), key=lambda x: x.label)+ async def _extract_unique_sources_and_files_from_agents(+ self, agent_states: List[AgentState], actor: User+ ) -> tuple[List[Source], List[FileMetadata]]:+ """Extract unique sources and files from agent states using bulk operations"""++ all_source_ids = set()+ all_file_ids = set()++ for agent_state in agent_states:+ files_agents = await self.file_agent_manager.list_files_for_agent(+ agent_id=agent_state.id, actor=actor, is_open_only=False, return_as_blocks=False+ )+ for file_agent in files_agents:+ all_source_ids.add(file_agent.source_id)+ all_file_ids.add(file_agent.file_id)+ sources = await self.source_manager.get_sources_by_ids_async(list(all_source_ids), actor)+ files = await self.file_manager.get_files_by_ids_async(list(all_file_ids), actor, include_content=True)++ return sources, files+async def _convert_agent_state_to_schema(self, agent_state: AgentState, actor: User) -> AgentSchema:"""Convert AgentState to AgentSchema with ID remapping"""agent_file_id = self._map_db_to_file_id(agent_state.id, AgentSchema.__id_prefix__)- agent_schema = await AgentSchema.from_agent_state(agent_state, message_manager=self.message_manager, actor=actor)+ files_agents = await self.file_agent_manager.list_files_for_agent(+ agent_id=agent_state.id, actor=actor, is_open_only=False, return_as_blocks=False+ )+ agent_schema = await AgentSchema.from_agent_state(+ agent_state, message_manager=self.message_manager, files_agents=files_agents, actor=actor+ )agent_schema.id = agent_file_id- # Remap message IDs and populate ID mapif agent_schema.messages:for message in agent_schema.messages:- # This call will populate the _db_to_file_ids mapping for this messagemessage_file_id = self._map_db_to_file_id(message.id, MessageSchema.__id_prefix__)- # Update the message's ID to the file IDmessage.id = message_file_id- # Update the message's agent_id to the file IDmessage.agent_id = agent_file_id- # Now remap in_context_message_ids using the populated mapif agent_schema.in_context_message_ids:agent_schema.in_context_message_ids = [self._map_db_to_file_id(message_id, MessageSchema.__id_prefix__, allow_new=False)for message_id in agent_schema.in_context_message_ids]- # Remap related entity IDs to file IDs using their schema prefixesif agent_schema.tool_ids:agent_schema.tool_ids = [self._map_db_to_file_id(tool_id, ToolSchema.__id_prefix__) for tool_id in agent_schema.tool_ids]- # TODO: Support sources- # if agent_schema.source_ids:- # agent_schema.source_ids = [- # self._map_db_to_file_id(source_id, SourceSchema.__id_prefix__) for source_id in agent_schema.source_ids- # ]- #+ if agent_schema.source_ids:+ agent_schema.source_ids = [+ self._map_db_to_file_id(source_id, SourceSchema.__id_prefix__) for source_id in agent_schema.source_ids+ ]+if agent_schema.block_ids:agent_schema.block_ids = [self._map_db_to_file_id(block_id, BlockSchema.__id_prefix__) for block_id in agent_schema.block_ids]- # TODO: Add other relationship ID mappings as needed- # (group_ids, etc.)+ if agent_schema.files_agents:+ for file_agent in agent_schema.files_agents:+ file_agent.file_id = self._map_db_to_file_id(file_agent.file_id, FileSchema.__id_prefix__)+ file_agent.source_id = self._map_db_to_file_id(file_agent.source_id, SourceSchema.__id_prefix__)+ file_agent.agent_id = agent_file_idreturn agent_schema@@ -190,6 +225,21 @@ class AgentFileManager:block_schema.id = block_file_idreturn block_schema+ def _convert_source_to_schema(self, source) -> SourceSchema:+ """Convert Source to SourceSchema with ID remapping"""+ source_file_id = self._map_db_to_file_id(source.id, SourceSchema.__id_prefix__, allow_new=False)+ source_schema = SourceSchema.from_source(source)+ source_schema.id = source_file_id+ return source_schema++ def _convert_file_to_schema(self, file_metadata) -> FileSchema:+ """Convert FileMetadata to FileSchema with ID remapping"""+ file_file_id = self._map_db_to_file_id(file_metadata.id, FileSchema.__id_prefix__, allow_new=False)+ file_schema = FileSchema.from_file_metadata(file_metadata)+ file_schema.id = file_file_id+ file_schema.source_id = self._map_db_to_file_id(file_metadata.source_id, SourceSchema.__id_prefix__, allow_new=False)+ return file_schema+async def export(self, agent_ids: List[str], actor: User) -> AgentFileSchema:"""Export agents and their related entities to AgentFileSchema format.@@ -218,10 +268,15 @@ class AgentFileManager:tool_set = self._extract_unique_tools(agent_states)block_set = self._extract_unique_blocks(agent_states)+ # Extract sources and files from agent states BEFORE conversion+ source_set, file_set = await self._extract_unique_sources_and_files_from_agents(agent_states, actor)+# Convert to schemas with ID remappingagent_schemas = [await self._convert_agent_state_to_schema(agent_state, actor=actor) for agent_state in agent_states]tool_schemas = [self._convert_tool_to_schema(tool) for tool in tool_set]block_schemas = [self._convert_block_to_schema(block) for block in block_set]+ source_schemas = [self._convert_source_to_schema(source) for source in source_set]+ file_schemas = [self._convert_file_to_schema(file_metadata) for file_metadata in file_set]logger.info(f"Exporting {len(agent_ids)} agents to agent file format")@@ -230,11 +285,12 @@ class AgentFileManager:agents=agent_schemas,groups=[], # TODO: Extract and convert groupsblocks=block_schemas,- files=[], # TODO: Extract and convert files- sources=[], # TODO: Extract and convert sources+ files=file_schemas,+ sources=source_schemas,tools=tool_schemas,# mcp_servers=[], # TODO: Extract and convert MCP serversmetadata={"revision_id": await get_latest_alembic_revision()},+ created_at=datetime.now(timezone.utc),)except Exception as e:@@ -294,7 +350,50 @@ class AgentFileManager:file_to_db_ids[block_schema.id] = created_block.idimported_count += 1- # 3. Create agents with empty message history+ # 3. Create sources (no dependencies)+ for source_schema in schema.sources:+ # Convert SourceSchema back to Source+ source_data = source_schema.model_dump(exclude={"id", "embedding", "embedding_chunk_size"})+ source = Source(**source_data)+ created_source = await self.source_manager.create_source(source, actor)+ file_to_db_ids[source_schema.id] = created_source.id+ imported_count += 1++ # 4. Create files (depends on sources)+ for file_schema in schema.files:+ # Convert FileSchema back to FileMetadata+ file_data = file_schema.model_dump(exclude={"id", "content"})+ # Remap source_id from file ID to database ID+ file_data["source_id"] = file_to_db_ids[file_schema.source_id]+ file_metadata = FileMetadata(**file_data)+ created_file = await self.file_manager.create_file(file_metadata, actor, text=file_schema.content)+ file_to_db_ids[file_schema.id] = created_file.id+ imported_count += 1++ # 5. Process files for chunking/embedding (depends on files and sources)+ file_processor = FileProcessor(+ file_parser=self.file_parser,+ embedder=self.embedder,+ actor=actor,+ using_pinecone=self.using_pinecone,+ )++ for file_schema in schema.files:+ if file_schema.content: # Only process files with content+ file_db_id = file_to_db_ids[file_schema.id]+ source_db_id = file_to_db_ids[file_schema.source_id]++ # Get the created file metadata+ file_metadata = await self.file_manager.get_file_by_id(file_db_id, actor)++ # Save the db call of fetching content again+ file_metadata.content = file_schema.content++ # Process the file for chunking/embedding+ passages = await file_processor.process_imported_file(file_metadata=file_metadata, source_id=source_db_id)+ imported_count += len(passages)++ # 6. Create agents with empty message historyfor agent_schema in schema.agents:# Convert AgentSchema back to CreateAgent, remapping tool/block IDsagent_data = agent_schema.model_dump(exclude={"id", "in_context_message_ids", "messages"})@@ -312,7 +411,7 @@ class AgentFileManager:file_to_db_ids[agent_schema.id] = created_agent.idimported_count += 1- # 4. Create messages and update agent message_ids+ # 7. Create messages and update agent message_idsfor agent_schema in schema.agents:agent_db_id = file_to_db_ids[agent_schema.id]message_file_to_db_ids = {}@@ -337,6 +436,29 @@ class AgentFileManager:# Update agent with the correct message_idsawait self.agent_manager.set_in_context_messages_async(agent_id=agent_db_id, message_ids=in_context_db_ids, actor=actor)+ # 8. Create file-agent relationships (depends on agents and files)+ for agent_schema in schema.agents:+ if agent_schema.files_agents:+ agent_db_id = file_to_db_ids[agent_schema.id]++ # Prepare files for bulk attachment+ files_for_agent = []+ visible_content_map = {}++ for file_agent_schema in agent_schema.files_agents:+ file_db_id = file_to_db_ids[file_agent_schema.file_id]+ file_metadata = await self.file_manager.get_file_by_id(file_db_id, actor)+ files_for_agent.append(file_metadata)++ if file_agent_schema.visible_content:+ visible_content_map[file_db_id] = file_agent_schema.visible_content++ # Bulk attach files to agent+ await self.file_agent_manager.attach_files_bulk(+ agent_id=agent_db_id, files_metadata=files_for_agent, visible_content_map=visible_content_map, actor=actor+ )+ imported_count += len(files_for_agent)+return ImportResult(success=True,message=f"Import completed successfully. Imported {imported_count} entities.",@@ -345,7 +467,7 @@ class AgentFileManager:)except Exception as e:- logger.error(f"Failed to import agent file: {e}")+ logger.exception(f"Failed to import agent file: {e}")raise AgentFileImportError(f"Import failed: {e}") from edef _validate_id_format(self, schema: AgentFileSchema) -> List[str]:@@ -435,6 +557,35 @@ class AgentFileManager:return errors+ def _validate_file_source_references(self, schema: AgentFileSchema) -> List[str]:+ """Validate that all file source_id references exist"""+ errors = []+ source_ids = {source.id for source in schema.sources}++ for file in schema.files:+ if file.source_id not in source_ids:+ errors.append(f"File {file.id} references non-existent source {file.source_id}")++ return errors++ def _validate_file_agent_references(self, schema: AgentFileSchema) -> List[str]:+ """Validate that all file-agent relationships reference existing entities"""+ errors = []+ file_ids = {file.id for file in schema.files}+ source_ids = {source.id for source in schema.sources}+ {agent.id for agent in schema.agents}++ for agent in schema.agents:+ for file_agent in agent.files_agents:+ if file_agent.file_id not in file_ids:+ errors.append(f"File-agent relationship references non-existent file {file_agent.file_id}")+ if file_agent.source_id not in source_ids:+ errors.append(f"File-agent relationship references non-existent source {file_agent.source_id}")+ if file_agent.agent_id != agent.id:+ errors.append(f"File-agent relationship has mismatched agent_id {file_agent.agent_id} vs {agent.id}")++ return errors+def _validate_schema(self, schema: AgentFileSchema):"""Validate the agent file schema for consistency and referential integrity.@@ -453,6 +604,12 @@ class AgentFileManager:# 2. Duplicate ID Detectionerrors.extend(self._validate_duplicate_ids(schema))+ # 3. File Source Reference Validation+ errors.extend(self._validate_file_source_references(schema))++ # 4. File-Agent Reference Validation+ errors.extend(self._validate_file_agent_references(schema))+if errors:raise AgentFileImportError(f"Schema validation failed: {'; '.join(errors)}")diff --git a/letta/services/file_manager.py b/letta/services/file_manager.pyindex 2d4c0d813..293354125 100644--- a/letta/services/file_manager.py+++ b/letta/services/file_manager.py@@ -1,3 +1,4 @@+import asyncioimport osfrom datetime import datetimefrom typing import List, Optional@@ -427,3 +428,83 @@ class FileManager:metadata.total_sources = len(source_aggregations)return metadata++ @enforce_types+ @trace_method+ async def get_files_by_ids_async(+ self, file_ids: List[str], actor: PydanticUser, *, include_content: bool = False+ ) -> List[PydanticFileMetadata]:+ """+ Get multiple files by their IDs in a single query.++ Args:+ file_ids: List of file IDs to retrieve+ actor: User performing the action+ include_content: Whether to include file content in the response++ Returns:+ List[PydanticFileMetadata]: List of files (may be fewer than requested if some don't exist)+ """+ if not file_ids:+ return []++ async with db_registry.async_session() as session:+ query = select(FileMetadataModel).where(+ FileMetadataModel.id.in_(file_ids),+ FileMetadataModel.organization_id == actor.organization_id,+ FileMetadataModel.is_deleted == False,+ )++ # Eagerly load content if requested+ if include_content:+ query = query.options(selectinload(FileMetadataModel.content))++ result = await session.execute(query)+ files_orm = result.scalars().all()++ return await asyncio.gather(*[file.to_pydantic_async(include_content=include_content) for file in files_orm])++ @enforce_types+ @trace_method+ async def get_files_for_agents_async(+ self, agent_ids: List[str], actor: PydanticUser, *, include_content: bool = False+ ) -> List[PydanticFileMetadata]:+ """+ Get all files associated with the given agents via file-agent relationships.++ Args:+ agent_ids: List of agent IDs to find files for+ actor: User performing the action+ include_content: Whether to include file content in the response++ Returns:+ List[PydanticFileMetadata]: List of unique files associated with these agents+ """+ if not agent_ids:+ return []++ async with db_registry.async_session() as session:+ # We need to import FileAgent here to avoid circular imports+ from letta.orm.file_agent import FileAgent as FileAgentModel++ # Join through file-agent relationships+ query = (+ select(FileMetadataModel)+ .join(FileAgentModel, FileMetadataModel.id == FileAgentModel.file_id)+ .where(+ FileAgentModel.agent_id.in_(agent_ids),+ FileMetadataModel.organization_id == actor.organization_id,+ FileMetadataModel.is_deleted == False,+ FileAgentModel.is_deleted == False,+ )+ .distinct() # Ensure we don't get duplicate files+ )++ # Eagerly load content if requested+ if include_content:+ query = query.options(selectinload(FileMetadataModel.content))++ result = await session.execute(query)+ files_orm = result.scalars().all()++ return await asyncio.gather(*[file.to_pydantic_async(include_content=include_content) for file in files_orm])diff --git a/letta/services/file_processor/file_processor.py b/letta/services/file_processor/file_processor.pyindex 4dec0ac22..09b0e51fc 100644--- a/letta/services/file_processor/file_processor.py+++ b/letta/services/file_processor/file_processor.py@@ -1,5 +1,7 @@from typing import List+from mistralai import OCRPageObject, OCRResponse, OCRUsageInfo+from letta.log import get_loggerfrom letta.otel.context import get_ctx_attributesfrom letta.otel.tracing import log_event, trace_method@@ -221,3 +223,94 @@ class FileProcessor:)return []++ def _create_ocr_response_from_content(self, content: str):+ """Create minimal OCR response from existing content"""+ return OCRResponse(+ model="import-skip-ocr",+ pages=[+ OCRPageObject(+ index=0,+ markdown=content,+ images=[],+ dimensions=None,+ )+ ],+ usage_info=OCRUsageInfo(pages_processed=1),+ document_annotation=None,+ )++ @trace_method+ async def process_imported_file(self, file_metadata: FileMetadata, source_id: str) -> List[Passage]:+ """Process an imported file that already has content - skip OCR, do chunking/embedding"""+ filename = file_metadata.file_name++ if not file_metadata.content:+ logger.warning(f"No content found for imported file {filename}")+ return []++ content = file_metadata.content+ try:+ # Create OCR response from existing content+ ocr_response = self._create_ocr_response_from_content(content)++ # Update file status to embedding+ file_metadata = await self.file_manager.update_file_status(+ file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.EMBEDDING+ )++ logger.info(f"Chunking imported file content for {filename}")+ log_event("file_processor.import_chunking_started", {"filename": filename, "content_length": len(content)})++ # Chunk and embed using existing logic+ all_passages = await self._chunk_and_embed_with_fallback(+ file_metadata=file_metadata, ocr_response=ocr_response, source_id=source_id+ )++ # Create passages in database (unless using Pinecone)+ if not self.using_pinecone:+ all_passages = await self.passage_manager.create_many_source_passages_async(+ passages=all_passages, file_metadata=file_metadata, actor=self.actor+ )+ log_event("file_processor.import_passages_created", {"filename": filename, "total_passages": len(all_passages)})++ # Update file status to completed+ if not self.using_pinecone:+ await self.file_manager.update_file_status(+ file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.COMPLETED+ )+ else:+ await self.file_manager.update_file_status(+ file_id=file_metadata.id, actor=self.actor, total_chunks=len(all_passages), chunks_embedded=0+ )++ logger.info(f"Successfully processed imported file {filename}: {len(all_passages)} passages")+ log_event(+ "file_processor.import_processing_completed",+ {+ "filename": filename,+ "file_id": str(file_metadata.id),+ "total_passages": len(all_passages),+ "status": FileProcessingStatus.COMPLETED.value,+ },+ )++ return all_passages++ except Exception as e:+ logger.exception("Import file processing failed for %s: %s", filename, e)+ log_event(+ "file_processor.import_processing_failed",+ {+ "filename": filename,+ "file_id": str(file_metadata.id),+ "error": str(e),+ "error_type": type(e).__name__,+ "status": FileProcessingStatus.ERROR.value,+ },+ )+ await self.file_manager.update_file_status(+ file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.ERROR, error_message=str(e)+ )++ return []diff --git a/letta/services/files_agents_manager.py b/letta/services/files_agents_manager.pyindex 0264f8dc8..9d8cc838f 100644--- a/letta/services/files_agents_manager.py+++ b/letta/services/files_agents_manager.py@@ -573,3 +573,31 @@ class FileAgentManager:if not assoc:raise NoResultFound(f"FileAgent(agent_id={agent_id}, file_name={file_name}) not found in org {actor.organization_id}")return assoc++ @enforce_types+ @trace_method+ async def get_files_agents_for_agents_async(self, agent_ids: List[str], actor: PydanticUser) -> List[PydanticFileAgent]:+ """+ Get all file-agent relationships for multiple agents in a single query.++ Args:+ agent_ids: List of agent IDs to find file-agent relationships for+ actor: User performing the action++ Returns:+ List[PydanticFileAgent]: List of file-agent relationships for these agents+ """+ if not agent_ids:+ return []++ async with db_registry.async_session() as session:+ query = select(FileAgentModel).where(+ FileAgentModel.agent_id.in_(agent_ids),+ FileAgentModel.organization_id == actor.organization_id,+ FileAgentModel.is_deleted == False,+ )++ result = await session.execute(query)+ file_agents_orm = result.scalars().all()++ return [file_agent.to_pydantic() for file_agent in file_agents_orm]diff --git a/letta/services/source_manager.py b/letta/services/source_manager.pyindex 02357c06c..e3287130c 100644--- a/letta/services/source_manager.py+++ b/letta/services/source_manager.py@@ -201,3 +201,63 @@ class SourceManager:return Noneelse:return sources[0].to_pydantic()++ @enforce_types+ @trace_method+ async def get_sources_by_ids_async(self, source_ids: List[str], actor: PydanticUser) -> List[PydanticSource]:+ """+ Get multiple sources by their IDs in a single query.++ Args:+ source_ids: List of source IDs to retrieve+ actor: User performing the action++ Returns:+ List[PydanticSource]: List of sources (may be fewer than requested if some don't exist)+ """+ if not source_ids:+ return []++ async with db_registry.async_session() as session:+ query = select(SourceModel).where(+ SourceModel.id.in_(source_ids), SourceModel.organization_id == actor.organization_id, SourceModel.is_deleted == False+ )++ result = await session.execute(query)+ sources_orm = result.scalars().all()++ return [source.to_pydantic() for source in sources_orm]++ @enforce_types+ @trace_method+ async def get_sources_for_agents_async(self, agent_ids: List[str], actor: PydanticUser) -> List[PydanticSource]:+ """+ Get all sources associated with the given agents via sources-agents relationships.++ Args:+ agent_ids: List of agent IDs to find sources for+ actor: User performing the action++ Returns:+ List[PydanticSource]: List of unique sources associated with these agents+ """+ if not agent_ids:+ return []++ async with db_registry.async_session() as session:+ # Join through sources-agents junction table+ query = (+ select(SourceModel)+ .join(SourcesAgents, SourceModel.id == SourcesAgents.source_id)+ .where(+ SourcesAgents.agent_id.in_(agent_ids),+ SourceModel.organization_id == actor.organization_id,+ SourceModel.is_deleted == False,+ )+ .distinct() # Ensure we don't get duplicate sources+ )++ result = await session.execute(query)+ sources_orm = result.scalars().all()++ return [source.to_pydantic() for source in sources_orm]diff --git a/tests/test_agent_serialization_v2.py b/tests/test_agent_serialization_v2.pyindex 462f2fc2d..4842a6589 100644--- a/tests/test_agent_serialization_v2.py+++ b/tests/test_agent_serialization_v2.py@@ -1,5 +1,5 @@import asyncio-from typing import List+from typing import List, Optionalimport pytest@@ -23,9 +23,12 @@ from letta.schemas.enums import MessageRolefrom letta.schemas.llm_config import LLMConfigfrom letta.schemas.message import MessageCreatefrom letta.schemas.organization import Organization+from letta.schemas.source import Sourcefrom letta.schemas.user import Userfrom letta.server.server import SyncServerfrom letta.services.agent_file_manager import AgentFileManager+from letta.services.file_processor.embedder.openai_embedder import OpenAIEmbedder+from letta.services.file_processor.parser.mistral_parser import MistralFileParserfrom tests.utils import create_tool_from_func# ------------------------------@@ -154,7 +157,7 @@ def test_block(server: SyncServer, default_user):@pytest.fixturedef agent_file_manager(server, default_user):- """Fixture to create AgentFileManager with all required services."""+ """Fixture to create AgentFileManager with all required services including file processing."""manager = AgentFileManager(agent_manager=server.agent_manager,tool_manager=server.tool_manager,@@ -165,6 +168,9 @@ def agent_file_manager(server, default_user):file_manager=server.file_manager,file_agent_manager=server.file_agent_manager,message_manager=server.message_manager,+ embedder=OpenAIEmbedder(),+ file_parser=MistralFileParser(),+ using_pinecone=False,)yield manager@@ -202,23 +208,149 @@ def test_agent(server: SyncServer, default_user, default_organization, test_blocactor=default_user,)- # Add more messages to create a richer conversation historyserver.send_messages(actor=default_user,agent_id=agent_state.id,input_messages=[MessageCreate(role=MessageRole.user, content="What's the weather like?")],)- # Get updated agent state with messagesagent_state = server.agent_manager.get_agent_by_id(agent_id=agent_state.id, actor=default_user)yield agent_state+async def test_source(server: SyncServer, default_user):+ """Fixture to create and return a test source."""+ source_data = Source(+ name="test_source",+ description="Test source for file export tests",+ embedding_config=EmbeddingConfig.default_config(provider="openai"),+ )+ source = await server.source_manager.create_source(source_data, default_user)+ yield source+++async def test_file(server: SyncServer, default_user, test_source):+ """Fixture to create and return a test file attached to test_source."""+ from letta.schemas.file import FileMetadata++ file_data = FileMetadata(+ source_id=test_source.id,+ file_name="test.txt",+ original_file_name="test.txt",+ file_type="text/plain",+ file_size=46,+ )+ file_metadata = await server.file_manager.create_file(file_data, default_user, text="This is a test file for export testing.")+ yield file_metadata+++async def agent_with_files(server: SyncServer, default_user, test_block, weather_tool, test_source, test_file):+ """Fixture to create and return an agent with attached files."""+ memory_blocks = [+ CreateBlock(label="human", value="User is a test user"),+ CreateBlock(label="persona", value="I am a helpful test assistant"),+ ]++ create_agent_request = CreateAgent(+ name="test_agent_v2",+ system="You are a helpful assistant for testing agent file export/import.",+ memory_blocks=memory_blocks,+ llm_config=LLMConfig.default_config("gpt-4o-mini"),+ embedding_config=EmbeddingConfig.default_config(provider="openai"),+ block_ids=[test_block.id],+ tool_ids=[weather_tool.id],+ tags=["test", "v2", "export"],+ description="Test agent for agent file v2 testing",+ metadata={"test_key": "test_value", "version": "v2"},+ initial_message_sequence=[+ MessageCreate(role=MessageRole.system, content="You are a helpful assistant."),+ MessageCreate(role=MessageRole.user, content="Hello!"),+ MessageCreate(role=MessageRole.assistant, content="Hello! How can I help you today?"),+ ],+ tool_exec_environment_variables={"TEST_VAR": "test_value"},+ message_buffer_autoclear=False,+ source_ids=[test_source.id],+ )++ agent_state = await server.agent_manager.create_agent_async(+ agent_create=create_agent_request,+ actor=default_user,+ )++ await server.insert_files_into_context_window(agent_state=agent_state, file_metadata_with_content=[test_file], actor=default_user)++ return (agent_state.id, test_source.id, test_file.id)++# ------------------------------# Helper Functions# ------------------------------+async def create_test_source(server: SyncServer, name: str, user: User):+ """Helper function to create a test source using server."""+ source_data = Source(+ name=name,+ description=f"Test source {name}",+ embedding_config=EmbeddingConfig.default_config(provider="openai"),+ )+ return await server.source_manager.create_source(source_data, user)+++async def create_test_file(server: SyncServer, filename: str, source_id: str, user: User, content: Optional[str] = None):+ """Helper function to create a test file using server."""+ from letta.schemas.file import FileMetadata++ content = content or f"Content of {filename}"+ file_data = FileMetadata(+ source_id=source_id,+ file_name=filename,+ original_file_name=filename,+ file_type="text/plain",+ file_size=len(content),+ )+ return await server.file_manager.create_file(file_data, user, text=content)+++async def create_test_agent_with_files(server: SyncServer, name: str, user: User, file_relationships: List[tuple]):+ """Helper function to create agent with attached files using server.++ Args:+ server: SyncServer instance+ name: Agent name+ user: User creating the agent+ file_relationships: List of (source_id, file_id) tuples+ """+ memory_blocks = [+ CreateBlock(label="human", value="User is a test user"),+ CreateBlock(label="persona", value="I am a helpful test assistant"),+ ]++ create_agent_request = CreateAgent(+ name=name,+ system="You are a helpful assistant for testing file export.",+ memory_blocks=memory_blocks,+ llm_config=LLMConfig.default_config("gpt-4o-mini"),+ embedding_config=EmbeddingConfig.default_config(provider="openai"),+ tags=["test", "files"],+ description="Test agent with files",+ )++ agent_state = await server.agent_manager.create_agent_async(+ agent_create=create_agent_request,+ actor=user,+ )++ for source_id, file_id in file_relationships:+ file_metadata = await server.file_manager.get_file_by_id(file_id, user)+ await server.insert_files_into_context_window(agent_state=agent_state, file_metadata_with_content=[file_metadata], actor=user)++ return agent_state++def compare_agent_files(original: AgentFileSchema, imported: AgentFileSchema) -> bool:"""Compare two AgentFileSchema objects for logical equivalence."""errors = []@@ -440,18 +572,25 @@ def _compare_files(orig: FileSchema, imp: FileSchema, index: int) -> List[str]:if orig.file_name != imp.file_name:errors.append(f"File {index}: file_name mismatch: '{orig.file_name}' vs '{imp.file_name}'")+ if orig.original_file_name != imp.original_file_name:+ errors.append(f"File {index}: original_file_name mismatch: '{orig.original_file_name}' vs '{imp.original_file_name}'")+if orig.file_size != imp.file_size:errors.append(f"File {index}: file_size mismatch: {orig.file_size} vs {imp.file_size}")if orig.file_type != imp.file_type:errors.append(f"File {index}: file_type mismatch: '{orig.file_type}' vs '{imp.file_type}'")- if orig.description != imp.description:- errors.append(f"File {index}: description mismatch")+ if orig.processing_status != imp.processing_status:+ errors.append(f"File {index}: processing_status mismatch: '{orig.processing_status}' vs '{imp.processing_status}'")if orig.metadata != imp.metadata:errors.append(f"File {index}: metadata mismatch")+ # Check source_id reference format (should be remapped)+ if not imp.source_id.startswith("source-"):+ errors.append(f"File {index}: source_id not properly remapped: {imp.source_id}")+return errors@@ -465,42 +604,61 @@ def _compare_sources(orig: SourceSchema, imp: SourceSchema, index: int) -> List[if orig.description != imp.description:errors.append(f"Source {index}: description mismatch")+ if orig.instructions != imp.instructions:+ errors.append(f"Source {index}: instructions mismatch")+if orig.metadata != imp.metadata:errors.append(f"Source {index}: metadata mismatch")+ if orig.embedding_config != imp.embedding_config:+ errors.append(f"Source {index}: embedding_config mismatch")+return errors+def _validate_entity_id(entity_id: str, expected_prefix: str) -> bool:+ """Helper function to validate that an ID follows the expected format (prefix-N)."""+ if not entity_id.startswith(f"{expected_prefix}-"):+ print(f"Invalid {expected_prefix} ID format: {entity_id} should start with '{expected_prefix}-'")+ return False++ try:+ suffix = entity_id[len(expected_prefix) + 1 :]+ int(suffix)+ return True+ except ValueError:+ print(f"Invalid {expected_prefix} ID format: {entity_id} should have integer suffix")+ return False++def validate_id_format(schema: AgentFileSchema) -> bool:"""Validate that all IDs follow the expected format (entity-N)."""- # Check agent IDsfor agent in schema.agents:- if not agent.id.startswith("agent-"):- print(f"Invalid agent ID format: {agent.id}")+ if not _validate_entity_id(agent.id, "agent"):return False- # Check message IDs within agentsfor message in agent.messages:- if not message.id.startswith("message-"):- print(f"Invalid message ID format: {message.id}")+ if not _validate_entity_id(message.id, "message"):return False- # Check in-context message ID referencesfor msg_id in agent.in_context_message_ids:- if not msg_id.startswith("message-"):- print(f"Invalid in-context message ID format: {msg_id}")+ if not _validate_entity_id(msg_id, "message"):return False- # Check tool IDsfor tool in schema.tools:- if not tool.id.startswith("tool-"):- print(f"Invalid tool ID format: {tool.id}")+ if not _validate_entity_id(tool.id, "tool"):return False- # Check block IDsfor block in schema.blocks:- if not block.id.startswith("block-"):- print(f"Invalid block ID format: {block.id}")+ if not _validate_entity_id(block.id, "block"):+ return False++ for file in schema.files:+ if not _validate_entity_id(file.id, "file"):+ return False++ for source in schema.sources:+ if not _validate_entity_id(source.id, "source"):return Falsereturn True@@ -511,29 +669,181 @@ def validate_id_format(schema: AgentFileSchema) -> bool:# ------------------------------+class TestFileExport:+ """Test file export functionality with comprehensive validation"""++ async def test_basic_file_export(self, default_user, agent_file_manager, agent_with_files):+ """Test basic file export functionality"""+ agent_id, source_id, file_id = agent_with_files++ exported = await agent_file_manager.export([agent_id], actor=default_user)++ assert len(exported.agents) == 1+ assert len(exported.sources) == 1+ assert len(exported.files) == 1++ agent = exported.agents[0]+ assert len(agent.files_agents) == 1++ assert _validate_entity_id(agent.id, "agent")+ assert _validate_entity_id(exported.sources[0].id, "source")+ assert _validate_entity_id(exported.files[0].id, "file")++ file_agent = agent.files_agents[0]+ assert file_agent.agent_id == agent.id+ assert file_agent.file_id == exported.files[0].id+ assert file_agent.source_id == exported.sources[0].id++ async def test_multiple_files_per_source(self, server, default_user, agent_file_manager):+ """Test export with multiple files from the same source"""+ source = await create_test_source(server, "multi-file-source", default_user)+ file1 = await create_test_file(server, "file1.txt", source.id, default_user)+ file2 = await create_test_file(server, "file2.txt", source.id, default_user)++ agent = await create_test_agent_with_files(server, "multi-file-agent", default_user, [(source.id, file1.id), (source.id, file2.id)])++ exported = await agent_file_manager.export([agent.id], actor=default_user)++ assert len(exported.agents) == 1+ assert len(exported.sources) == 1+ assert len(exported.files) == 2++ agent = exported.agents[0]+ assert len(agent.files_agents) == 2++ source_id = exported.sources[0].id+ for file_schema in exported.files:+ assert file_schema.source_id == source_id++ file_ids = {f.id for f in exported.files}+ for file_agent in agent.files_agents:+ assert file_agent.file_id in file_ids+ assert file_agent.source_id == source_id++ async def test_multiple_sources_export(self, server, default_user, agent_file_manager):+ """Test export with files from multiple sources"""+ source1 = await create_test_source(server, "source-1", default_user)+ source2 = await create_test_source(server, "source-2", default_user)+ file1 = await create_test_file(server, "file1.txt", source1.id, default_user)+ file2 = await create_test_file(server, "file2.txt", source2.id, default_user)++ agent = await create_test_agent_with_files(+ server, "multi-source-agent", default_user, [(source1.id, file1.id), (source2.id, file2.id)]+ )++ exported = await agent_file_manager.export([agent.id], actor=default_user)++ assert len(exported.agents) == 1+ assert len(exported.sources) == 2+ assert len(exported.files) == 2++ source_ids = {s.id for s in exported.sources}+ for file_schema in exported.files:+ assert file_schema.source_id in source_ids++ async def test_cross_agent_file_deduplication(self, server, default_user, agent_file_manager):+ """Test that files shared across agents are deduplicated in export"""+ source = await create_test_source(server, "shared-source", default_user)+ shared_file = await create_test_file(server, "shared.txt", source.id, default_user)++ agent1 = await create_test_agent_with_files(server, "agent-1", default_user, [(source.id, shared_file.id)])+ agent2 = await create_test_agent_with_files(server, "agent-2", default_user, [(source.id, shared_file.id)])++ exported = await agent_file_manager.export([agent1.id, agent2.id], actor=default_user)++ assert len(exported.agents) == 2+ assert len(exported.sources) == 1+ assert len(exported.files) == 1++ file_id = exported.files[0].id+ source_id = exported.sources[0].id++ for agent in exported.agents:+ assert len(agent.files_agents) == 1+ file_agent = agent.files_agents[0]+ assert file_agent.file_id == file_id+ assert file_agent.source_id == source_id++ async def test_file_agent_relationship_preservation(self, server, default_user, agent_file_manager):+ """Test that file-agent relationship details are preserved"""+ source = await create_test_source(server, "test-source", default_user)+ file = await create_test_file(server, "test.txt", source.id, default_user)++ agent = await create_test_agent_with_files(server, "test-agent", default_user, [(source.id, file.id)])++ exported = await agent_file_manager.export([agent.id], actor=default_user)++ agent = exported.agents[0]+ file_agent = agent.files_agents[0]++ assert file_agent.file_name == file.file_name+ assert file_agent.is_open is True+ assert hasattr(file_agent, "last_accessed_at")++ async def test_id_remapping_consistency(self, server, default_user, agent_file_manager):+ """Test that ID remapping is consistent across all references"""+ source = await create_test_source(server, "consistency-source", default_user)+ file = await create_test_file(server, "consistency.txt", source.id, default_user)+ agent = await create_test_agent_with_files(server, "consistency-agent", default_user, [(source.id, file.id)])++ exported = await agent_file_manager.export([agent.id], actor=default_user)++ agent_schema = exported.agents[0]+ source_schema = exported.sources[0]+ file_schema = exported.files[0]+ file_agent = agent_schema.files_agents[0]++ assert file_schema.source_id == source_schema.id+ assert file_agent.agent_id == agent_schema.id+ assert file_agent.file_id == file_schema.id+ assert file_agent.source_id == source_schema.id++ async def test_empty_file_relationships(self, server, default_user, agent_file_manager):+ """Test export of agent with no file relationships"""+ agent_create = CreateAgent(+ name="no-files-agent",+ llm_config=LLMConfig.default_config("gpt-4o-mini"),+ embedding_config=EmbeddingConfig.default_config(provider="openai"),+ )+ agent = await server.agent_manager.create_agent_async(agent_create, actor=default_user)++ exported = await agent_file_manager.export([agent.id], actor=default_user)++ assert len(exported.agents) == 1+ assert len(exported.sources) == 0+ assert len(exported.files) == 0++ agent_schema = exported.agents[0]+ assert len(agent_schema.files_agents) == 0++ async def test_file_content_inclusion_in_export(self, default_user, agent_file_manager, agent_with_files):+ """Test that file content is included in export"""+ agent_id, source_id, file_id = agent_with_files++ exported = await agent_file_manager.export([agent_id], actor=default_user)++ file_schema = exported.files[0]+ assert hasattr(file_schema, "content") or file_schema.content is not None++class TestAgentFileExport:"""Tests for agent file export functionality."""async def test_basic_export(self, agent_file_manager, test_agent, default_user):"""Test basic agent export functionality."""- # Export the agentagent_file = await agent_file_manager.export([test_agent.id], default_user)- # Validate the structureassert isinstance(agent_file, AgentFileSchema)assert len(agent_file.agents) == 1assert len(agent_file.tools) > 0 # Should include base tools + weather toolassert len(agent_file.blocks) > 0 # Should include memory blocks + test block- # Validate revision_id is automatically set in metadataassert agent_file.metadata.get("revision_id") is not Noneassert agent_file.metadata.get("revision_id") != "unknown"assert len(agent_file.metadata.get("revision_id")) > 0- # Validate ID formatsassert validate_id_format(agent_file)- # Check agent dataexported_agent = agent_file.agents[0]assert exported_agent.name == test_agent.nameassert exported_agent.system == test_agent.system@@ -542,7 +852,6 @@ class TestAgentFileExport:async def test_export_multiple_agents(self, server, agent_file_manager, test_agent, default_user, weather_tool):"""Test exporting multiple agents."""- # Create a second agentcreate_agent_request = CreateAgent(name="second_test_agent",system="Second test agent",@@ -559,29 +868,23 @@ class TestAgentFileExport:actor=default_user,)- # Export both agentsagent_file = await agent_file_manager.export([test_agent.id, second_agent.id], default_user)- # Validateassert len(agent_file.agents) == 2assert validate_id_format(agent_file)- # Check that agents have different IDs but tools are deduplicatedagent_ids = {agent.id for agent in agent_file.agents}assert len(agent_ids) == 2async def test_export_id_remapping(self, agent_file_manager, test_agent, default_user):"""Test that IDs are properly remapped during export."""- # Export the agentagent_file = await agent_file_manager.export([test_agent.id], default_user)exported_agent = agent_file.agents[0]- # Verify agent ID is remappedassert exported_agent.id == "agent-0"assert exported_agent.id != test_agent.id- # Verify tool/block IDs are remappedif exported_agent.tool_ids:for tool_id in exported_agent.tool_ids:assert tool_id.startswith("tool-")@@ -590,25 +893,21 @@ class TestAgentFileExport:for block_id in exported_agent.block_ids:assert block_id.startswith("block-")- # Verify message IDs are remapped and in-context references are consistentmessage_ids = {msg.id for msg in exported_agent.messages}for in_context_id in exported_agent.in_context_message_ids:assert in_context_id in message_ids, f"In-context message ID {in_context_id} not found in messages"async def test_message_agent_id_remapping(self, agent_file_manager, test_agent, default_user):"""Test that message.agent_id is properly remapped during export."""- # Export the agentagent_file = await agent_file_manager.export([test_agent.id], default_user)exported_agent = agent_file.agents[0]- # Verify all messages have the remapped agent_id matching the exported agentfor message in exported_agent.messages:assert (message.agent_id == exported_agent.id), f"Message {message.id} has agent_id {message.agent_id}, expected {exported_agent.id}"- # Verify agent_id is the remapped file ID formatassert exported_agent.id == "agent-0"assert exported_agent.id != test_agent.id@@ -627,21 +926,16 @@ class TestAgentFileExport:async def test_revision_id_automatic_setting(self, agent_file_manager, test_agent, default_user):"""Test that revision_id is automatically set to the latest alembic revision."""- # Export the agentagent_file = await agent_file_manager.export([test_agent.id], default_user)- # Get the expected revision ID from the functionfrom letta.utils import get_latest_alembic_revisionexpected_revision = await get_latest_alembic_revision()- # Validate that the revision_id matches the latest alembic revisionassert agent_file.metadata.get("revision_id") == expected_revision- # Validate that it's not the fallback "unknown" valueassert agent_file.metadata.get("revision_id") != "unknown"- # Validate that it looks like a valid revision ID (12 hex characters)assert len(agent_file.metadata.get("revision_id")) == 12assert all(c in "0123456789abcdef" for c in agent_file.metadata.get("revision_id"))@@ -651,52 +945,41 @@ class TestAgentFileImport:async def test_basic_import(self, agent_file_manager, test_agent, default_user, other_user):"""Test basic agent import functionality."""- # Export the agentagent_file = await agent_file_manager.export([test_agent.id], default_user)- # Import the agentresult = await agent_file_manager.import_file(agent_file, other_user)- # Validate import resultassert result.successassert result.imported_count > 0assert len(result.id_mappings) > 0- # Verify new entities were created (not existing ones reused)for file_id, db_id in result.id_mappings.items():if file_id.startswith("agent-"):assert db_id != test_agent.id # New agent should have different IDasync def test_import_preserves_data(self, server, agent_file_manager, test_agent, default_user, other_user):"""Test that import preserves all important data."""- # Export the agentagent_file = await agent_file_manager.export([test_agent.id], default_user)- # Import the agentresult = await agent_file_manager.import_file(agent_file, other_user)- # Get the imported agentimported_agent_id = next(db_id for file_id, db_id in result.id_mappings.items() if file_id == "agent-0")imported_agent = server.agent_manager.get_agent_by_id(imported_agent_id, other_user)- # Compare key fieldsassert imported_agent.name == test_agent.nameassert imported_agent.system == test_agent.systemassert imported_agent.description == test_agent.descriptionassert imported_agent.metadata == test_agent.metadataassert imported_agent.tags == test_agent.tags- # Check that tools and blocks were importedassert len(imported_agent.tools) == len(test_agent.tools)assert len(imported_agent.memory.blocks) == len(test_agent.memory.blocks)- # Check that messages were importedoriginal_messages = server.message_manager.list_messages_for_agent(test_agent.id, default_user)imported_messages = server.message_manager.list_messages_for_agent(imported_agent_id, other_user)assert len(imported_messages) == len(original_messages)- # Verify message content is preservedfor orig_msg, imp_msg in zip(original_messages, imported_messages):assert orig_msg.role == imp_msg.roleassert orig_msg.content == imp_msg.content@@ -704,20 +987,15 @@ class TestAgentFileImport:async def test_import_message_context_preservation(self, server, agent_file_manager, test_agent, default_user, other_user):"""Test that in-context message references are preserved during import."""- # Export the agentagent_file = await agent_file_manager.export([test_agent.id], default_user)- # Import the agentresult = await agent_file_manager.import_file(agent_file, other_user)- # Get the imported agentimported_agent_id = next(db_id for file_id, db_id in result.id_mappings.items() if file_id == "agent-0")imported_agent = server.agent_manager.get_agent_by_id(imported_agent_id, other_user)- # Check that in-context message count is preservedassert len(imported_agent.message_ids) == len(test_agent.message_ids)- # Verify all in-context messages existimported_messages = server.message_manager.list_messages_for_agent(imported_agent_id, other_user)imported_message_ids = {msg.id for msg in imported_messages}@@ -726,13 +1004,10 @@ class TestAgentFileImport:async def test_dry_run_import(self, agent_file_manager, test_agent, default_user, other_user):"""Test dry run import validation."""- # Export the agentagent_file = await agent_file_manager.export([test_agent.id], default_user)- # Dry run importresult = await agent_file_manager.import_file(agent_file, other_user, dry_run=True)- # Validate dry run resultassert result.successassert result.imported_count == 0 # No actual imports in dry runassert len(result.id_mappings) == 0@@ -740,12 +1015,10 @@ class TestAgentFileImport:async def test_import_validation_errors(self, agent_file_manager, other_user):"""Test import validation catches errors."""- # Get current revision for testfrom letta.utils import get_latest_alembic_revisioncurrent_revision = await get_latest_alembic_revision()- # Create invalid agent file with duplicate IDsinvalid_agent_file = AgentFileSchema(metadata={"revision_id": current_revision},agents=[@@ -759,29 +1032,101 @@ class TestAgentFileImport:tools=[],)- # Import should fail validationwith pytest.raises(AgentFileImportError):await agent_file_manager.import_file(invalid_agent_file, other_user)+class TestAgentFileImportWithProcessing:+ """Tests for agent file import with file processing (chunking/embedding)."""++ async def test_import_with_file_processing(self, server, agent_file_manager, default_user, other_user):+ """Test that import processes files for chunking and embedding."""+ source = await create_test_source(server, "processing-source", default_user)+ file_content = "This is test content for processing. It should be chunked and embedded during import."+ file_metadata = await create_test_file(server, "process.txt", source.id, default_user, content=file_content)++ agent = await create_test_agent_with_files(server, "processing-agent", default_user, [(source.id, file_metadata.id)])++ exported = await agent_file_manager.export([agent.id], default_user)++ result = await agent_file_manager.import_file(exported, other_user)++ assert result.success+ assert result.imported_count > 0++ imported_file_id = next(db_id for file_id, db_id in result.id_mappings.items() if file_id.startswith("file-"))++ imported_file = await server.file_manager.get_file_by_id(imported_file_id, other_user)+ assert imported_file.processing_status.value == "completed"++ async def test_import_passage_creation(self, server, agent_file_manager, default_user, other_user):+ """Test that import creates passages for file content."""+ source = await create_test_source(server, "passage-source", default_user)+ file_content = "This content should create passages. Each sentence should be chunked separately."+ file_metadata = await create_test_file(server, "passages.txt", source.id, default_user, content=file_content)++ agent = await create_test_agent_with_files(server, "passage-agent", default_user, [(source.id, file_metadata.id)])++ exported = await agent_file_manager.export([agent.id], default_user)++ result = await agent_file_manager.import_file(exported, other_user)++ imported_file_id = next(db_id for file_id, db_id in result.id_mappings.items() if file_id.startswith("file-"))++ passages = await server.passage_manager.list_passages_by_file_id_async(imported_file_id, other_user)+ assert len(passages) > 0++ for passage in passages:+ assert passage.embedding is not None+ assert len(passage.embedding) > 0++ async def test_import_file_status_updates(self, server, agent_file_manager, default_user, other_user):+ """Test that file processing status is updated correctly during import."""+ source = await create_test_source(server, "status-source", default_user)+ file_metadata = await create_test_file(server, "status.txt", source.id, default_user)++ agent = await create_test_agent_with_files(server, "status-agent", default_user, [(source.id, file_metadata.id)])++ exported = await agent_file_manager.export([agent.id], default_user)++ result = await agent_file_manager.import_file(exported, other_user)++ imported_file_id = next(db_id for file_id, db_id in result.id_mappings.items() if file_id.startswith("file-"))+ imported_file = await server.file_manager.get_file_by_id(imported_file_id, other_user)++ assert imported_file.processing_status.value == "completed"+ assert imported_file.total_chunks is None+ assert imported_file.chunks_embedded is None++ async def test_import_multiple_files_processing(self, server, agent_file_manager, default_user, other_user):+ """Test import processes multiple files efficiently."""+ source = await create_test_source(server, "multi-source", default_user)+ file1 = await create_test_file(server, "file1.txt", source.id, default_user)+ file2 = await create_test_file(server, "file2.txt", source.id, default_user)++ agent = await create_test_agent_with_files(server, "multi-agent", default_user, [(source.id, file1.id), (source.id, file2.id)])++ exported = await agent_file_manager.export([agent.id], default_user)++ result = await agent_file_manager.import_file(exported, other_user)++ imported_file_ids = [db_id for file_id, db_id in result.id_mappings.items() if file_id.startswith("file-")]+ assert len(imported_file_ids) == 2++ for file_id in imported_file_ids:+ imported_file = await server.file_manager.get_file_by_id(file_id, other_user)+ assert imported_file.processing_status.value == "completed"++class TestAgentFileRoundTrip:"""Tests for complete export -> import -> export cycles."""async def test_roundtrip_consistency(self, server, agent_file_manager, test_agent, default_user, other_user):"""Test that export -> import -> export produces consistent results."""- # First exportoriginal_export = await agent_file_manager.export([test_agent.id], default_user)-- # Importresult = await agent_file_manager.import_file(original_export, other_user)-- # Get imported agent IDimported_agent_id = next(db_id for file_id, db_id in result.id_mappings.items() if file_id == "agent-0")-- # Second exportsecond_export = await agent_file_manager.export([imported_agent_id], other_user)-- # Compare exports (should be logically equivalent)assert compare_agent_files(original_export, second_export)async def test_multiple_roundtrips(self, server, agent_file_manager, test_agent, default_user, other_user):@@ -790,18 +1135,14 @@ class TestAgentFileRoundTrip:current_user = default_userfor i in range(3):- # Exportagent_file = await agent_file_manager.export([current_agent_id], current_user)- # Import to other usertarget_user = other_user if current_user == default_user else default_userresult = await agent_file_manager.import_file(agent_file, target_user)- # Update for next iterationcurrent_agent_id = next(db_id for file_id, db_id in result.id_mappings.items() if file_id == "agent-0")current_user = target_user- # Verify the agent still worksimported_agent = server.agent_manager.get_agent_by_id(current_agent_id, current_user)assert imported_agent.name == test_agent.name