Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat: Support exporting/importing sources and files for agent file (#3414)

Matthew Zhoub0138ecad695
diff --git a/letta/orm/organization.py b/letta/orm/organization.py
index f5f65cb92..4f78ec895 100644
--- a/letta/orm/organization.py
+++ b/letta/orm/organization.py
@@ -6,18 +6,17 @@ from letta.orm.sqlalchemy_base import SqlalchemyBase
from letta.schemas.organization import Organization as PydanticOrganization
if TYPE_CHECKING:
+ from letta.orm import Source
from letta.orm.agent import Agent
- from letta.orm.agent_passage import AgentPassage
from letta.orm.block import Block
from letta.orm.group import Group
from letta.orm.identity import Identity
- from letta.orm.llm_batch_item import LLMBatchItem
+ from letta.orm.llm_batch_items import LLMBatchItem
from letta.orm.llm_batch_job import LLMBatchJob
from letta.orm.message import Message
+ from letta.orm.passage import AgentPassage, SourcePassage
from letta.orm.provider import Provider
- from letta.orm.sandbox_config import AgentEnvironmentVariable, SandboxConfig
- from letta.orm.sandbox_environment_variable import SandboxEnvironmentVariable
- from letta.orm.source_passage import SourcePassage
+ from letta.orm.sandbox_config import AgentEnvironmentVariable, SandboxConfig, SandboxEnvironmentVariable
from letta.orm.tool import Tool
from letta.orm.user import User
@@ -48,6 +47,7 @@ class Organization(SqlalchemyBase):
# relationships
agents: Mapped[List["Agent"]] = relationship("Agent", back_populates="organization", cascade="all, delete-orphan")
+ sources: Mapped[List["Source"]] = relationship("Source", cascade="all, delete-orphan")
messages: Mapped[List["Message"]] = relationship("Message", back_populates="organization", cascade="all, delete-orphan")
source_passages: Mapped[List["SourcePassage"]] = relationship(
"SourcePassage", back_populates="organization", cascade="all, delete-orphan"
diff --git a/letta/schemas/agent_file.py b/letta/schemas/agent_file.py
index ae669b921..c9a18072f 100644
--- a/letta/schemas/agent_file.py
+++ b/letta/schemas/agent_file.py
@@ -1,14 +1,15 @@
+from datetime import datetime
from typing import Dict, List, Optional
from pydantic import BaseModel, Field
from letta.schemas.agent import AgentState, CreateAgent
-from letta.schemas.block import CreateBlock
+from letta.schemas.block import Block, CreateBlock
from letta.schemas.enums import MessageRole
-from letta.schemas.file import FileMetadataCreate
+from letta.schemas.file import FileAgent, FileAgentBase, FileMetadata, FileMetadataBase
from letta.schemas.group import GroupCreate
from letta.schemas.message import Message, MessageCreate
-from letta.schemas.source import SourceCreate
+from letta.schemas.source import Source, SourceCreate
from letta.schemas.tool import Tool
from letta.schemas.user import User
from letta.services.message_manager import MessageManager
@@ -63,6 +64,30 @@ class MessageSchema(MessageCreate):
)
+class FileAgentSchema(FileAgentBase):
+ """File-Agent relationship with human-readable ID for agent file"""
+
+ __id_prefix__ = "file_agent"
+ id: str = Field(..., description="Human-readable identifier for this file-agent relationship in the file")
+
+ @classmethod
+ def from_file_agent(cls, file_agent: FileAgent) -> "FileAgentSchema":
+ """Convert FileAgent to FileAgentSchema"""
+
+ create_file_agent = FileAgentBase(
+ agent_id=file_agent.agent_id,
+ file_id=file_agent.file_id,
+ source_id=file_agent.source_id,
+ file_name=file_agent.file_name,
+ is_open=file_agent.is_open,
+ visible_content=file_agent.visible_content,
+ last_accessed_at=file_agent.last_accessed_at,
+ )
+
+ # Create FileAgentSchema with the file_agent's ID (will be remapped later)
+ return cls(id=file_agent.id, **create_file_agent.model_dump())
+
+
class AgentSchema(CreateAgent):
"""Agent with human-readable ID for agent file"""
@@ -72,9 +97,12 @@ class AgentSchema(CreateAgent):
default_factory=list, description="List of message IDs that are currently in the agent's context"
)
messages: List[MessageSchema] = Field(default_factory=list, description="List of messages in the agent's conversation history")
+ files_agents: List[FileAgentSchema] = Field(default_factory=list, description="List of file-agent relationships for this agent")
@classmethod
- async def from_agent_state(cls, agent_state: AgentState, message_manager: MessageManager, actor: User) -> "AgentSchema":
+ async def from_agent_state(
+ cls, agent_state: AgentState, message_manager: MessageManager, files_agents: List[FileAgent], actor: User
+ ) -> "AgentSchema":
"""Convert AgentState to AgentSchema"""
create_agent = CreateAgent(
@@ -131,6 +159,7 @@ class AgentSchema(CreateAgent):
id=agent_state.id,
in_context_message_ids=agent_state.message_ids or [],
messages=message_schemas, # Messages will be populated separately by the manager
+ files_agents=[FileAgentSchema.from_file_agent(f) for f in files_agents],
**create_agent.model_dump(),
)
@@ -149,7 +178,7 @@ class BlockSchema(CreateBlock):
id: str = Field(..., description="Human-readable identifier for this block in the file")
@classmethod
- def from_block(cls, block) -> "BlockSchema":
+ def from_block(cls, block: Block) -> "BlockSchema":
"""Convert Block to BlockSchema"""
create_block = CreateBlock(
@@ -168,12 +197,35 @@ class BlockSchema(CreateBlock):
return cls(id=block.id, **create_block.model_dump())
-class FileSchema(FileMetadataCreate):
+class FileSchema(FileMetadataBase):
"""File with human-readable ID for agent file"""
__id_prefix__ = "file"
id: str = Field(..., description="Human-readable identifier for this file in the file")
+ @classmethod
+ def from_file_metadata(cls, file_metadata: FileMetadata) -> "FileSchema":
+ """Convert FileMetadata to FileSchema"""
+
+ create_file = FileMetadataBase(
+ source_id=file_metadata.source_id,
+ file_name=file_metadata.file_name,
+ original_file_name=file_metadata.original_file_name,
+ file_path=file_metadata.file_path,
+ file_type=file_metadata.file_type,
+ file_size=file_metadata.file_size,
+ file_creation_date=file_metadata.file_creation_date,
+ file_last_modified_date=file_metadata.file_last_modified_date,
+ processing_status=file_metadata.processing_status,
+ error_message=file_metadata.error_message,
+ total_chunks=file_metadata.total_chunks,
+ chunks_embedded=file_metadata.chunks_embedded,
+ content=file_metadata.content,
+ )
+
+ # Create FileSchema with the file's ID (will be remapped later)
+ return cls(id=file_metadata.id, **create_file.model_dump())
+
class SourceSchema(SourceCreate):
"""Source with human-readable ID for agent file"""
@@ -181,6 +233,21 @@ class SourceSchema(SourceCreate):
__id_prefix__ = "source"
id: str = Field(..., description="Human-readable identifier for this source in the file")
+ @classmethod
+ def from_source(cls, source: Source) -> "SourceSchema":
+ """Convert Block to BlockSchema"""
+
+ create_block = SourceCreate(
+ name=source.name,
+ description=source.description,
+ instructions=source.instructions,
+ metadata=source.metadata,
+ embedding_config=source.embedding_config,
+ )
+
+ # Create SourceSchema with the block's ID (will be remapped later)
+ return cls(id=source.id, **create_block.model_dump())
+
# TODO: This one is quite thin, just a wrapper over Tool
class ToolSchema(Tool):
@@ -200,13 +267,6 @@ class ToolSchema(Tool):
# id: str = Field(..., description="Human-readable identifier for this MCP server in the file")
-# class FileAgentStateSchema(FileAgentCreate):
-# """File-Agent relationship with human-readable ID for agent file"""
-#
-# __id_prefix__ = "file_agent"
-# id: str = Field(..., description="Human-readable identifier for this file-agent relationship in the file")
-
-
class AgentFileSchema(BaseModel):
"""Schema for serialized agent file that can be exported to JSON and imported into agent server."""
@@ -220,3 +280,4 @@ class AgentFileSchema(BaseModel):
metadata: Dict[str, str] = Field(
default_factory=dict, description="Metadata for this agent file, including revision_id and other export information."
)
+ created_at: Optional[datetime] = Field(default=None, description="The timestamp when the object was created.")
diff --git a/letta/schemas/file.py b/letta/schemas/file.py
index b74da074b..4a3c876f8 100644
--- a/letta/schemas/file.py
+++ b/letta/schemas/file.py
@@ -74,6 +74,10 @@ class FileAgentBase(LettaBase):
None,
description="Portion of the file the agent is focused on (may be large).",
)
+ last_accessed_at: Optional[datetime] = Field(
+ default_factory=datetime.utcnow,
+ description="UTC timestamp of the agent’s most recent access to this file.",
+ )
class FileAgent(FileAgentBase):
@@ -94,10 +98,6 @@ class FileAgent(FileAgentBase):
None,
description="Org ID this association belongs to (inherited from both agent and file).",
)
- last_accessed_at: Optional[datetime] = Field(
- default_factory=datetime.utcnow,
- description="UTC timestamp of the agent’s most recent access to this file.",
- )
created_at: Optional[datetime] = Field(
default_factory=datetime.utcnow,
@@ -108,11 +108,3 @@ class FileAgent(FileAgentBase):
description="Row last-update timestamp (UTC).",
)
is_deleted: bool = Field(False, description="Soft-delete flag.")
-
-
-class FileMetadataCreate(FileMetadataBase):
- """Schema for creating a new FileMetadata"""
-
-
-class FileAgentCreate(FileAgentBase):
- """Schema for creating a new FileAgent association"""
diff --git a/letta/services/agent_file_manager.py b/letta/services/agent_file_manager.py
index d58c3d418..001cd964d 100644
--- a/letta/services/agent_file_manager.py
+++ b/letta/services/agent_file_manager.py
@@ -1,3 +1,4 @@
+from datetime import datetime, timezone
from typing import Dict, List
from letta.errors import AgentFileExportError, AgentFileImportError
@@ -7,6 +8,7 @@ from letta.schemas.agent_file import (
AgentFileSchema,
AgentSchema,
BlockSchema,
+ FileAgentSchema,
FileSchema,
GroupSchema,
ImportResult,
@@ -15,12 +17,17 @@ from letta.schemas.agent_file import (
ToolSchema,
)
from letta.schemas.block import Block
+from letta.schemas.file import FileMetadata
from letta.schemas.message import Message
+from letta.schemas.source import Source
from letta.schemas.tool import Tool
from letta.schemas.user import User
from letta.services.agent_manager import AgentManager
from letta.services.block_manager import BlockManager
from letta.services.file_manager import FileManager
+from letta.services.file_processor.embedder.base_embedder import BaseEmbedder
+from letta.services.file_processor.file_processor import FileProcessor
+from letta.services.file_processor.parser.mistral_parser import MistralFileParser
from letta.services.files_agents_manager import FileAgentManager
from letta.services.group_manager import GroupManager
from letta.services.mcp_manager import MCPManager
@@ -54,6 +61,9 @@ class AgentFileManager:
file_manager: FileManager,
file_agent_manager: FileAgentManager,
message_manager: MessageManager,
+ embedder: BaseEmbedder,
+ file_parser: MistralFileParser,
+ using_pinecone: bool = False,
):
self.agent_manager = agent_manager
self.tool_manager = tool_manager
@@ -64,6 +74,9 @@ class AgentFileManager:
self.file_manager = file_manager
self.file_agent_manager = file_agent_manager
self.message_manager = message_manager
+ self.embedder = embedder
+ self.file_parser = file_parser
+ self.using_pinecone = using_pinecone
# ID mapping state for export
self._db_to_file_ids: Dict[str, str] = {}
@@ -77,6 +90,7 @@ class AgentFileManager:
SourceSchema.__id_prefix__: 0,
ToolSchema.__id_prefix__: 0,
MessageSchema.__id_prefix__: 0,
+ FileAgentSchema.__id_prefix__: 0,
# MCPServerSchema.__id_prefix__: 0,
}
@@ -134,45 +148,66 @@ class AgentFileManager:
return sorted(unique_blocks.values(), key=lambda x: x.label)
+ async def _extract_unique_sources_and_files_from_agents(
+ self, agent_states: List[AgentState], actor: User
+ ) -> tuple[List[Source], List[FileMetadata]]:
+ """Extract unique sources and files from agent states using bulk operations"""
+
+ all_source_ids = set()
+ all_file_ids = set()
+
+ for agent_state in agent_states:
+ files_agents = await self.file_agent_manager.list_files_for_agent(
+ agent_id=agent_state.id, actor=actor, is_open_only=False, return_as_blocks=False
+ )
+ for file_agent in files_agents:
+ all_source_ids.add(file_agent.source_id)
+ all_file_ids.add(file_agent.file_id)
+ sources = await self.source_manager.get_sources_by_ids_async(list(all_source_ids), actor)
+ files = await self.file_manager.get_files_by_ids_async(list(all_file_ids), actor, include_content=True)
+
+ return sources, files
+
async def _convert_agent_state_to_schema(self, agent_state: AgentState, actor: User) -> AgentSchema:
"""Convert AgentState to AgentSchema with ID remapping"""
agent_file_id = self._map_db_to_file_id(agent_state.id, AgentSchema.__id_prefix__)
- agent_schema = await AgentSchema.from_agent_state(agent_state, message_manager=self.message_manager, actor=actor)
+ files_agents = await self.file_agent_manager.list_files_for_agent(
+ agent_id=agent_state.id, actor=actor, is_open_only=False, return_as_blocks=False
+ )
+ agent_schema = await AgentSchema.from_agent_state(
+ agent_state, message_manager=self.message_manager, files_agents=files_agents, actor=actor
+ )
agent_schema.id = agent_file_id
- # Remap message IDs and populate ID map
if agent_schema.messages:
for message in agent_schema.messages:
- # This call will populate the _db_to_file_ids mapping for this message
message_file_id = self._map_db_to_file_id(message.id, MessageSchema.__id_prefix__)
- # Update the message's ID to the file ID
message.id = message_file_id
- # Update the message's agent_id to the file ID
message.agent_id = agent_file_id
- # Now remap in_context_message_ids using the populated map
if agent_schema.in_context_message_ids:
agent_schema.in_context_message_ids = [
self._map_db_to_file_id(message_id, MessageSchema.__id_prefix__, allow_new=False)
for message_id in agent_schema.in_context_message_ids
]
- # Remap related entity IDs to file IDs using their schema prefixes
if agent_schema.tool_ids:
agent_schema.tool_ids = [self._map_db_to_file_id(tool_id, ToolSchema.__id_prefix__) for tool_id in agent_schema.tool_ids]
- # TODO: Support sources
- # if agent_schema.source_ids:
- # agent_schema.source_ids = [
- # self._map_db_to_file_id(source_id, SourceSchema.__id_prefix__) for source_id in agent_schema.source_ids
- # ]
- #
+ if agent_schema.source_ids:
+ agent_schema.source_ids = [
+ self._map_db_to_file_id(source_id, SourceSchema.__id_prefix__) for source_id in agent_schema.source_ids
+ ]
+
if agent_schema.block_ids:
agent_schema.block_ids = [self._map_db_to_file_id(block_id, BlockSchema.__id_prefix__) for block_id in agent_schema.block_ids]
- # TODO: Add other relationship ID mappings as needed
- # (group_ids, etc.)
+ if agent_schema.files_agents:
+ for file_agent in agent_schema.files_agents:
+ file_agent.file_id = self._map_db_to_file_id(file_agent.file_id, FileSchema.__id_prefix__)
+ file_agent.source_id = self._map_db_to_file_id(file_agent.source_id, SourceSchema.__id_prefix__)
+ file_agent.agent_id = agent_file_id
return agent_schema
@@ -190,6 +225,21 @@ class AgentFileManager:
block_schema.id = block_file_id
return block_schema
+ def _convert_source_to_schema(self, source) -> SourceSchema:
+ """Convert Source to SourceSchema with ID remapping"""
+ source_file_id = self._map_db_to_file_id(source.id, SourceSchema.__id_prefix__, allow_new=False)
+ source_schema = SourceSchema.from_source(source)
+ source_schema.id = source_file_id
+ return source_schema
+
+ def _convert_file_to_schema(self, file_metadata) -> FileSchema:
+ """Convert FileMetadata to FileSchema with ID remapping"""
+ file_file_id = self._map_db_to_file_id(file_metadata.id, FileSchema.__id_prefix__, allow_new=False)
+ file_schema = FileSchema.from_file_metadata(file_metadata)
+ file_schema.id = file_file_id
+ file_schema.source_id = self._map_db_to_file_id(file_metadata.source_id, SourceSchema.__id_prefix__, allow_new=False)
+ return file_schema
+
async def export(self, agent_ids: List[str], actor: User) -> AgentFileSchema:
"""
Export agents and their related entities to AgentFileSchema format.
@@ -218,10 +268,15 @@ class AgentFileManager:
tool_set = self._extract_unique_tools(agent_states)
block_set = self._extract_unique_blocks(agent_states)
+ # Extract sources and files from agent states BEFORE conversion
+ source_set, file_set = await self._extract_unique_sources_and_files_from_agents(agent_states, actor)
+
# Convert to schemas with ID remapping
agent_schemas = [await self._convert_agent_state_to_schema(agent_state, actor=actor) for agent_state in agent_states]
tool_schemas = [self._convert_tool_to_schema(tool) for tool in tool_set]
block_schemas = [self._convert_block_to_schema(block) for block in block_set]
+ source_schemas = [self._convert_source_to_schema(source) for source in source_set]
+ file_schemas = [self._convert_file_to_schema(file_metadata) for file_metadata in file_set]
logger.info(f"Exporting {len(agent_ids)} agents to agent file format")
@@ -230,11 +285,12 @@ class AgentFileManager:
agents=agent_schemas,
groups=[], # TODO: Extract and convert groups
blocks=block_schemas,
- files=[], # TODO: Extract and convert files
- sources=[], # TODO: Extract and convert sources
+ files=file_schemas,
+ sources=source_schemas,
tools=tool_schemas,
# mcp_servers=[], # TODO: Extract and convert MCP servers
metadata={"revision_id": await get_latest_alembic_revision()},
+ created_at=datetime.now(timezone.utc),
)
except Exception as e:
@@ -294,7 +350,50 @@ class AgentFileManager:
file_to_db_ids[block_schema.id] = created_block.id
imported_count += 1
- # 3. Create agents with empty message history
+ # 3. Create sources (no dependencies)
+ for source_schema in schema.sources:
+ # Convert SourceSchema back to Source
+ source_data = source_schema.model_dump(exclude={"id", "embedding", "embedding_chunk_size"})
+ source = Source(**source_data)
+ created_source = await self.source_manager.create_source(source, actor)
+ file_to_db_ids[source_schema.id] = created_source.id
+ imported_count += 1
+
+ # 4. Create files (depends on sources)
+ for file_schema in schema.files:
+ # Convert FileSchema back to FileMetadata
+ file_data = file_schema.model_dump(exclude={"id", "content"})
+ # Remap source_id from file ID to database ID
+ file_data["source_id"] = file_to_db_ids[file_schema.source_id]
+ file_metadata = FileMetadata(**file_data)
+ created_file = await self.file_manager.create_file(file_metadata, actor, text=file_schema.content)
+ file_to_db_ids[file_schema.id] = created_file.id
+ imported_count += 1
+
+ # 5. Process files for chunking/embedding (depends on files and sources)
+ file_processor = FileProcessor(
+ file_parser=self.file_parser,
+ embedder=self.embedder,
+ actor=actor,
+ using_pinecone=self.using_pinecone,
+ )
+
+ for file_schema in schema.files:
+ if file_schema.content: # Only process files with content
+ file_db_id = file_to_db_ids[file_schema.id]
+ source_db_id = file_to_db_ids[file_schema.source_id]
+
+ # Get the created file metadata
+ file_metadata = await self.file_manager.get_file_by_id(file_db_id, actor)
+
+ # Save the db call of fetching content again
+ file_metadata.content = file_schema.content
+
+ # Process the file for chunking/embedding
+ passages = await file_processor.process_imported_file(file_metadata=file_metadata, source_id=source_db_id)
+ imported_count += len(passages)
+
+ # 6. Create agents with empty message history
for agent_schema in schema.agents:
# Convert AgentSchema back to CreateAgent, remapping tool/block IDs
agent_data = agent_schema.model_dump(exclude={"id", "in_context_message_ids", "messages"})
@@ -312,7 +411,7 @@ class AgentFileManager:
file_to_db_ids[agent_schema.id] = created_agent.id
imported_count += 1
- # 4. Create messages and update agent message_ids
+ # 7. Create messages and update agent message_ids
for agent_schema in schema.agents:
agent_db_id = file_to_db_ids[agent_schema.id]
message_file_to_db_ids = {}
@@ -337,6 +436,29 @@ class AgentFileManager:
# Update agent with the correct message_ids
await self.agent_manager.set_in_context_messages_async(agent_id=agent_db_id, message_ids=in_context_db_ids, actor=actor)
+ # 8. Create file-agent relationships (depends on agents and files)
+ for agent_schema in schema.agents:
+ if agent_schema.files_agents:
+ agent_db_id = file_to_db_ids[agent_schema.id]
+
+ # Prepare files for bulk attachment
+ files_for_agent = []
+ visible_content_map = {}
+
+ for file_agent_schema in agent_schema.files_agents:
+ file_db_id = file_to_db_ids[file_agent_schema.file_id]
+ file_metadata = await self.file_manager.get_file_by_id(file_db_id, actor)
+ files_for_agent.append(file_metadata)
+
+ if file_agent_schema.visible_content:
+ visible_content_map[file_db_id] = file_agent_schema.visible_content
+
+ # Bulk attach files to agent
+ await self.file_agent_manager.attach_files_bulk(
+ agent_id=agent_db_id, files_metadata=files_for_agent, visible_content_map=visible_content_map, actor=actor
+ )
+ imported_count += len(files_for_agent)
+
return ImportResult(
success=True,
message=f"Import completed successfully. Imported {imported_count} entities.",
@@ -345,7 +467,7 @@ class AgentFileManager:
)
except Exception as e:
- logger.error(f"Failed to import agent file: {e}")
+ logger.exception(f"Failed to import agent file: {e}")
raise AgentFileImportError(f"Import failed: {e}") from e
def _validate_id_format(self, schema: AgentFileSchema) -> List[str]:
@@ -435,6 +557,35 @@ class AgentFileManager:
return errors
+ def _validate_file_source_references(self, schema: AgentFileSchema) -> List[str]:
+ """Validate that all file source_id references exist"""
+ errors = []
+ source_ids = {source.id for source in schema.sources}
+
+ for file in schema.files:
+ if file.source_id not in source_ids:
+ errors.append(f"File {file.id} references non-existent source {file.source_id}")
+
+ return errors
+
+ def _validate_file_agent_references(self, schema: AgentFileSchema) -> List[str]:
+ """Validate that all file-agent relationships reference existing entities"""
+ errors = []
+ file_ids = {file.id for file in schema.files}
+ source_ids = {source.id for source in schema.sources}
+ {agent.id for agent in schema.agents}
+
+ for agent in schema.agents:
+ for file_agent in agent.files_agents:
+ if file_agent.file_id not in file_ids:
+ errors.append(f"File-agent relationship references non-existent file {file_agent.file_id}")
+ if file_agent.source_id not in source_ids:
+ errors.append(f"File-agent relationship references non-existent source {file_agent.source_id}")
+ if file_agent.agent_id != agent.id:
+ errors.append(f"File-agent relationship has mismatched agent_id {file_agent.agent_id} vs {agent.id}")
+
+ return errors
+
def _validate_schema(self, schema: AgentFileSchema):
"""
Validate the agent file schema for consistency and referential integrity.
@@ -453,6 +604,12 @@ class AgentFileManager:
# 2. Duplicate ID Detection
errors.extend(self._validate_duplicate_ids(schema))
+ # 3. File Source Reference Validation
+ errors.extend(self._validate_file_source_references(schema))
+
+ # 4. File-Agent Reference Validation
+ errors.extend(self._validate_file_agent_references(schema))
+
if errors:
raise AgentFileImportError(f"Schema validation failed: {'; '.join(errors)}")
diff --git a/letta/services/file_manager.py b/letta/services/file_manager.py
index 2d4c0d813..293354125 100644
--- a/letta/services/file_manager.py
+++ b/letta/services/file_manager.py
@@ -1,3 +1,4 @@
+import asyncio
import os
from datetime import datetime
from typing import List, Optional
@@ -427,3 +428,83 @@ class FileManager:
metadata.total_sources = len(source_aggregations)
return metadata
+
+ @enforce_types
+ @trace_method
+ async def get_files_by_ids_async(
+ self, file_ids: List[str], actor: PydanticUser, *, include_content: bool = False
+ ) -> List[PydanticFileMetadata]:
+ """
+ Get multiple files by their IDs in a single query.
+
+ Args:
+ file_ids: List of file IDs to retrieve
+ actor: User performing the action
+ include_content: Whether to include file content in the response
+
+ Returns:
+ List[PydanticFileMetadata]: List of files (may be fewer than requested if some don't exist)
+ """
+ if not file_ids:
+ return []
+
+ async with db_registry.async_session() as session:
+ query = select(FileMetadataModel).where(
+ FileMetadataModel.id.in_(file_ids),
+ FileMetadataModel.organization_id == actor.organization_id,
+ FileMetadataModel.is_deleted == False,
+ )
+
+ # Eagerly load content if requested
+ if include_content:
+ query = query.options(selectinload(FileMetadataModel.content))
+
+ result = await session.execute(query)
+ files_orm = result.scalars().all()
+
+ return await asyncio.gather(*[file.to_pydantic_async(include_content=include_content) for file in files_orm])
+
+ @enforce_types
+ @trace_method
+ async def get_files_for_agents_async(
+ self, agent_ids: List[str], actor: PydanticUser, *, include_content: bool = False
+ ) -> List[PydanticFileMetadata]:
+ """
+ Get all files associated with the given agents via file-agent relationships.
+
+ Args:
+ agent_ids: List of agent IDs to find files for
+ actor: User performing the action
+ include_content: Whether to include file content in the response
+
+ Returns:
+ List[PydanticFileMetadata]: List of unique files associated with these agents
+ """
+ if not agent_ids:
+ return []
+
+ async with db_registry.async_session() as session:
+ # We need to import FileAgent here to avoid circular imports
+ from letta.orm.file_agent import FileAgent as FileAgentModel
+
+ # Join through file-agent relationships
+ query = (
+ select(FileMetadataModel)
+ .join(FileAgentModel, FileMetadataModel.id == FileAgentModel.file_id)
+ .where(
+ FileAgentModel.agent_id.in_(agent_ids),
+ FileMetadataModel.organization_id == actor.organization_id,
+ FileMetadataModel.is_deleted == False,
+ FileAgentModel.is_deleted == False,
+ )
+ .distinct() # Ensure we don't get duplicate files
+ )
+
+ # Eagerly load content if requested
+ if include_content:
+ query = query.options(selectinload(FileMetadataModel.content))
+
+ result = await session.execute(query)
+ files_orm = result.scalars().all()
+
+ return await asyncio.gather(*[file.to_pydantic_async(include_content=include_content) for file in files_orm])
diff --git a/letta/services/file_processor/file_processor.py b/letta/services/file_processor/file_processor.py
index 4dec0ac22..09b0e51fc 100644
--- a/letta/services/file_processor/file_processor.py
+++ b/letta/services/file_processor/file_processor.py
@@ -1,5 +1,7 @@
from typing import List
+from mistralai import OCRPageObject, OCRResponse, OCRUsageInfo
+
from letta.log import get_logger
from letta.otel.context import get_ctx_attributes
from letta.otel.tracing import log_event, trace_method
@@ -221,3 +223,94 @@ class FileProcessor:
)
return []
+
+ def _create_ocr_response_from_content(self, content: str):
+ """Create minimal OCR response from existing content"""
+ return OCRResponse(
+ model="import-skip-ocr",
+ pages=[
+ OCRPageObject(
+ index=0,
+ markdown=content,
+ images=[],
+ dimensions=None,
+ )
+ ],
+ usage_info=OCRUsageInfo(pages_processed=1),
+ document_annotation=None,
+ )
+
+ @trace_method
+ async def process_imported_file(self, file_metadata: FileMetadata, source_id: str) -> List[Passage]:
+ """Process an imported file that already has content - skip OCR, do chunking/embedding"""
+ filename = file_metadata.file_name
+
+ if not file_metadata.content:
+ logger.warning(f"No content found for imported file {filename}")
+ return []
+
+ content = file_metadata.content
+ try:
+ # Create OCR response from existing content
+ ocr_response = self._create_ocr_response_from_content(content)
+
+ # Update file status to embedding
+ file_metadata = await self.file_manager.update_file_status(
+ file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.EMBEDDING
+ )
+
+ logger.info(f"Chunking imported file content for {filename}")
+ log_event("file_processor.import_chunking_started", {"filename": filename, "content_length": len(content)})
+
+ # Chunk and embed using existing logic
+ all_passages = await self._chunk_and_embed_with_fallback(
+ file_metadata=file_metadata, ocr_response=ocr_response, source_id=source_id
+ )
+
+ # Create passages in database (unless using Pinecone)
+ if not self.using_pinecone:
+ all_passages = await self.passage_manager.create_many_source_passages_async(
+ passages=all_passages, file_metadata=file_metadata, actor=self.actor
+ )
+ log_event("file_processor.import_passages_created", {"filename": filename, "total_passages": len(all_passages)})
+
+ # Update file status to completed
+ if not self.using_pinecone:
+ await self.file_manager.update_file_status(
+ file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.COMPLETED
+ )
+ else:
+ await self.file_manager.update_file_status(
+ file_id=file_metadata.id, actor=self.actor, total_chunks=len(all_passages), chunks_embedded=0
+ )
+
+ logger.info(f"Successfully processed imported file {filename}: {len(all_passages)} passages")
+ log_event(
+ "file_processor.import_processing_completed",
+ {
+ "filename": filename,
+ "file_id": str(file_metadata.id),
+ "total_passages": len(all_passages),
+ "status": FileProcessingStatus.COMPLETED.value,
+ },
+ )
+
+ return all_passages
+
+ except Exception as e:
+ logger.exception("Import file processing failed for %s: %s", filename, e)
+ log_event(
+ "file_processor.import_processing_failed",
+ {
+ "filename": filename,
+ "file_id": str(file_metadata.id),
+ "error": str(e),
+ "error_type": type(e).__name__,
+ "status": FileProcessingStatus.ERROR.value,
+ },
+ )
+ await self.file_manager.update_file_status(
+ file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.ERROR, error_message=str(e)
+ )
+
+ return []
diff --git a/letta/services/files_agents_manager.py b/letta/services/files_agents_manager.py
index 0264f8dc8..9d8cc838f 100644
--- a/letta/services/files_agents_manager.py
+++ b/letta/services/files_agents_manager.py
@@ -573,3 +573,31 @@ class FileAgentManager:
if not assoc:
raise NoResultFound(f"FileAgent(agent_id={agent_id}, file_name={file_name}) not found in org {actor.organization_id}")
return assoc
+
+ @enforce_types
+ @trace_method
+ async def get_files_agents_for_agents_async(self, agent_ids: List[str], actor: PydanticUser) -> List[PydanticFileAgent]:
+ """
+ Get all file-agent relationships for multiple agents in a single query.
+
+ Args:
+ agent_ids: List of agent IDs to find file-agent relationships for
+ actor: User performing the action
+
+ Returns:
+ List[PydanticFileAgent]: List of file-agent relationships for these agents
+ """
+ if not agent_ids:
+ return []
+
+ async with db_registry.async_session() as session:
+ query = select(FileAgentModel).where(
+ FileAgentModel.agent_id.in_(agent_ids),
+ FileAgentModel.organization_id == actor.organization_id,
+ FileAgentModel.is_deleted == False,
+ )
+
+ result = await session.execute(query)
+ file_agents_orm = result.scalars().all()
+
+ return [file_agent.to_pydantic() for file_agent in file_agents_orm]
diff --git a/letta/services/source_manager.py b/letta/services/source_manager.py
index 02357c06c..e3287130c 100644
--- a/letta/services/source_manager.py
+++ b/letta/services/source_manager.py
@@ -201,3 +201,63 @@ class SourceManager:
return None
else:
return sources[0].to_pydantic()
+
+ @enforce_types
+ @trace_method
+ async def get_sources_by_ids_async(self, source_ids: List[str], actor: PydanticUser) -> List[PydanticSource]:
+ """
+ Get multiple sources by their IDs in a single query.
+
+ Args:
+ source_ids: List of source IDs to retrieve
+ actor: User performing the action
+
+ Returns:
+ List[PydanticSource]: List of sources (may be fewer than requested if some don't exist)
+ """
+ if not source_ids:
+ return []
+
+ async with db_registry.async_session() as session:
+ query = select(SourceModel).where(
+ SourceModel.id.in_(source_ids), SourceModel.organization_id == actor.organization_id, SourceModel.is_deleted == False
+ )
+
+ result = await session.execute(query)
+ sources_orm = result.scalars().all()
+
+ return [source.to_pydantic() for source in sources_orm]
+
+ @enforce_types
+ @trace_method
+ async def get_sources_for_agents_async(self, agent_ids: List[str], actor: PydanticUser) -> List[PydanticSource]:
+ """
+ Get all sources associated with the given agents via sources-agents relationships.
+
+ Args:
+ agent_ids: List of agent IDs to find sources for
+ actor: User performing the action
+
+ Returns:
+ List[PydanticSource]: List of unique sources associated with these agents
+ """
+ if not agent_ids:
+ return []
+
+ async with db_registry.async_session() as session:
+ # Join through sources-agents junction table
+ query = (
+ select(SourceModel)
+ .join(SourcesAgents, SourceModel.id == SourcesAgents.source_id)
+ .where(
+ SourcesAgents.agent_id.in_(agent_ids),
+ SourceModel.organization_id == actor.organization_id,
+ SourceModel.is_deleted == False,
+ )
+ .distinct() # Ensure we don't get duplicate sources
+ )
+
+ result = await session.execute(query)
+ sources_orm = result.scalars().all()
+
+ return [source.to_pydantic() for source in sources_orm]
diff --git a/tests/test_agent_serialization_v2.py b/tests/test_agent_serialization_v2.py
index 462f2fc2d..4842a6589 100644
--- a/tests/test_agent_serialization_v2.py
+++ b/tests/test_agent_serialization_v2.py
@@ -1,5 +1,5 @@
import asyncio
-from typing import List
+from typing import List, Optional
import pytest
@@ -23,9 +23,12 @@ from letta.schemas.enums import MessageRole
from letta.schemas.llm_config import LLMConfig
from letta.schemas.message import MessageCreate
from letta.schemas.organization import Organization
+from letta.schemas.source import Source
from letta.schemas.user import User
from letta.server.server import SyncServer
from letta.services.agent_file_manager import AgentFileManager
+from letta.services.file_processor.embedder.openai_embedder import OpenAIEmbedder
+from letta.services.file_processor.parser.mistral_parser import MistralFileParser
from tests.utils import create_tool_from_func
# ------------------------------
@@ -154,7 +157,7 @@ def test_block(server: SyncServer, default_user):
@pytest.fixture
def agent_file_manager(server, default_user):
- """Fixture to create AgentFileManager with all required services."""
+ """Fixture to create AgentFileManager with all required services including file processing."""
manager = AgentFileManager(
agent_manager=server.agent_manager,
tool_manager=server.tool_manager,
@@ -165,6 +168,9 @@ def agent_file_manager(server, default_user):
file_manager=server.file_manager,
file_agent_manager=server.file_agent_manager,
message_manager=server.message_manager,
+ embedder=OpenAIEmbedder(),
+ file_parser=MistralFileParser(),
+ using_pinecone=False,
)
yield manager
@@ -202,23 +208,149 @@ def test_agent(server: SyncServer, default_user, default_organization, test_bloc
actor=default_user,
)
- # Add more messages to create a richer conversation history
server.send_messages(
actor=default_user,
agent_id=agent_state.id,
input_messages=[MessageCreate(role=MessageRole.user, content="What's the weather like?")],
)
- # Get updated agent state with messages
agent_state = server.agent_manager.get_agent_by_id(agent_id=agent_state.id, actor=default_user)
yield agent_state
+async def test_source(server: SyncServer, default_user):
+ """Fixture to create and return a test source."""
+ source_data = Source(
+ name="test_source",
+ description="Test source for file export tests",
+ embedding_config=EmbeddingConfig.default_config(provider="openai"),
+ )
+ source = await server.source_manager.create_source(source_data, default_user)
+ yield source
+
+
+async def test_file(server: SyncServer, default_user, test_source):
+ """Fixture to create and return a test file attached to test_source."""
+ from letta.schemas.file import FileMetadata
+
+ file_data = FileMetadata(
+ source_id=test_source.id,
+ file_name="test.txt",
+ original_file_name="test.txt",
+ file_type="text/plain",
+ file_size=46,
+ )
+ file_metadata = await server.file_manager.create_file(file_data, default_user, text="This is a test file for export testing.")
+ yield file_metadata
+
+
+async def agent_with_files(server: SyncServer, default_user, test_block, weather_tool, test_source, test_file):
+ """Fixture to create and return an agent with attached files."""
+ memory_blocks = [
+ CreateBlock(label="human", value="User is a test user"),
+ CreateBlock(label="persona", value="I am a helpful test assistant"),
+ ]
+
+ create_agent_request = CreateAgent(
+ name="test_agent_v2",
+ system="You are a helpful assistant for testing agent file export/import.",
+ memory_blocks=memory_blocks,
+ llm_config=LLMConfig.default_config("gpt-4o-mini"),
+ embedding_config=EmbeddingConfig.default_config(provider="openai"),
+ block_ids=[test_block.id],
+ tool_ids=[weather_tool.id],
+ tags=["test", "v2", "export"],
+ description="Test agent for agent file v2 testing",
+ metadata={"test_key": "test_value", "version": "v2"},
+ initial_message_sequence=[
+ MessageCreate(role=MessageRole.system, content="You are a helpful assistant."),
+ MessageCreate(role=MessageRole.user, content="Hello!"),
+ MessageCreate(role=MessageRole.assistant, content="Hello! How can I help you today?"),
+ ],
+ tool_exec_environment_variables={"TEST_VAR": "test_value"},
+ message_buffer_autoclear=False,
+ source_ids=[test_source.id],
+ )
+
+ agent_state = await server.agent_manager.create_agent_async(
+ agent_create=create_agent_request,
+ actor=default_user,
+ )
+
+ await server.insert_files_into_context_window(agent_state=agent_state, file_metadata_with_content=[test_file], actor=default_user)
+
+ return (agent_state.id, test_source.id, test_file.id)
+
+
# ------------------------------
# Helper Functions
# ------------------------------
+async def create_test_source(server: SyncServer, name: str, user: User):
+ """Helper function to create a test source using server."""
+ source_data = Source(
+ name=name,
+ description=f"Test source {name}",
+ embedding_config=EmbeddingConfig.default_config(provider="openai"),
+ )
+ return await server.source_manager.create_source(source_data, user)
+
+
+async def create_test_file(server: SyncServer, filename: str, source_id: str, user: User, content: Optional[str] = None):
+ """Helper function to create a test file using server."""
+ from letta.schemas.file import FileMetadata
+
+ content = content or f"Content of {filename}"
+ file_data = FileMetadata(
+ source_id=source_id,
+ file_name=filename,
+ original_file_name=filename,
+ file_type="text/plain",
+ file_size=len(content),
+ )
+ return await server.file_manager.create_file(file_data, user, text=content)
+
+
+async def create_test_agent_with_files(server: SyncServer, name: str, user: User, file_relationships: List[tuple]):
+ """Helper function to create agent with attached files using server.
+
+ Args:
+ server: SyncServer instance
+ name: Agent name
+ user: User creating the agent
+ file_relationships: List of (source_id, file_id) tuples
+ """
+ memory_blocks = [
+ CreateBlock(label="human", value="User is a test user"),
+ CreateBlock(label="persona", value="I am a helpful test assistant"),
+ ]
+
+ create_agent_request = CreateAgent(
+ name=name,
+ system="You are a helpful assistant for testing file export.",
+ memory_blocks=memory_blocks,
+ llm_config=LLMConfig.default_config("gpt-4o-mini"),
+ embedding_config=EmbeddingConfig.default_config(provider="openai"),
+ tags=["test", "files"],
+ description="Test agent with files",
+ )
+
+ agent_state = await server.agent_manager.create_agent_async(
+ agent_create=create_agent_request,
+ actor=user,
+ )
+
+ for source_id, file_id in file_relationships:
+ file_metadata = await server.file_manager.get_file_by_id(file_id, user)
+ await server.insert_files_into_context_window(agent_state=agent_state, file_metadata_with_content=[file_metadata], actor=user)
+
+ return agent_state
+
+
def compare_agent_files(original: AgentFileSchema, imported: AgentFileSchema) -> bool:
"""Compare two AgentFileSchema objects for logical equivalence."""
errors = []
@@ -440,18 +572,25 @@ def _compare_files(orig: FileSchema, imp: FileSchema, index: int) -> List[str]:
if orig.file_name != imp.file_name:
errors.append(f"File {index}: file_name mismatch: '{orig.file_name}' vs '{imp.file_name}'")
+ if orig.original_file_name != imp.original_file_name:
+ errors.append(f"File {index}: original_file_name mismatch: '{orig.original_file_name}' vs '{imp.original_file_name}'")
+
if orig.file_size != imp.file_size:
errors.append(f"File {index}: file_size mismatch: {orig.file_size} vs {imp.file_size}")
if orig.file_type != imp.file_type:
errors.append(f"File {index}: file_type mismatch: '{orig.file_type}' vs '{imp.file_type}'")
- if orig.description != imp.description:
- errors.append(f"File {index}: description mismatch")
+ if orig.processing_status != imp.processing_status:
+ errors.append(f"File {index}: processing_status mismatch: '{orig.processing_status}' vs '{imp.processing_status}'")
if orig.metadata != imp.metadata:
errors.append(f"File {index}: metadata mismatch")
+ # Check source_id reference format (should be remapped)
+ if not imp.source_id.startswith("source-"):
+ errors.append(f"File {index}: source_id not properly remapped: {imp.source_id}")
+
return errors
@@ -465,42 +604,61 @@ def _compare_sources(orig: SourceSchema, imp: SourceSchema, index: int) -> List[
if orig.description != imp.description:
errors.append(f"Source {index}: description mismatch")
+ if orig.instructions != imp.instructions:
+ errors.append(f"Source {index}: instructions mismatch")
+
if orig.metadata != imp.metadata:
errors.append(f"Source {index}: metadata mismatch")
+ if orig.embedding_config != imp.embedding_config:
+ errors.append(f"Source {index}: embedding_config mismatch")
+
return errors
+def _validate_entity_id(entity_id: str, expected_prefix: str) -> bool:
+ """Helper function to validate that an ID follows the expected format (prefix-N)."""
+ if not entity_id.startswith(f"{expected_prefix}-"):
+ print(f"Invalid {expected_prefix} ID format: {entity_id} should start with '{expected_prefix}-'")
+ return False
+
+ try:
+ suffix = entity_id[len(expected_prefix) + 1 :]
+ int(suffix)
+ return True
+ except ValueError:
+ print(f"Invalid {expected_prefix} ID format: {entity_id} should have integer suffix")
+ return False
+
+
def validate_id_format(schema: AgentFileSchema) -> bool:
"""Validate that all IDs follow the expected format (entity-N)."""
- # Check agent IDs
for agent in schema.agents:
- if not agent.id.startswith("agent-"):
- print(f"Invalid agent ID format: {agent.id}")
+ if not _validate_entity_id(agent.id, "agent"):
return False
- # Check message IDs within agents
for message in agent.messages:
- if not message.id.startswith("message-"):
- print(f"Invalid message ID format: {message.id}")
+ if not _validate_entity_id(message.id, "message"):
return False
- # Check in-context message ID references
for msg_id in agent.in_context_message_ids:
- if not msg_id.startswith("message-"):
- print(f"Invalid in-context message ID format: {msg_id}")
+ if not _validate_entity_id(msg_id, "message"):
return False
- # Check tool IDs
for tool in schema.tools:
- if not tool.id.startswith("tool-"):
- print(f"Invalid tool ID format: {tool.id}")
+ if not _validate_entity_id(tool.id, "tool"):
return False
- # Check block IDs
for block in schema.blocks:
- if not block.id.startswith("block-"):
- print(f"Invalid block ID format: {block.id}")
+ if not _validate_entity_id(block.id, "block"):
+ return False
+
+ for file in schema.files:
+ if not _validate_entity_id(file.id, "file"):
+ return False
+
+ for source in schema.sources:
+ if not _validate_entity_id(source.id, "source"):
return False
return True
@@ -511,29 +669,181 @@ def validate_id_format(schema: AgentFileSchema) -> bool:
# ------------------------------
+class TestFileExport:
+ """Test file export functionality with comprehensive validation"""
+
+ async def test_basic_file_export(self, default_user, agent_file_manager, agent_with_files):
+ """Test basic file export functionality"""
+ agent_id, source_id, file_id = agent_with_files
+
+ exported = await agent_file_manager.export([agent_id], actor=default_user)
+
+ assert len(exported.agents) == 1
+ assert len(exported.sources) == 1
+ assert len(exported.files) == 1
+
+ agent = exported.agents[0]
+ assert len(agent.files_agents) == 1
+
+ assert _validate_entity_id(agent.id, "agent")
+ assert _validate_entity_id(exported.sources[0].id, "source")
+ assert _validate_entity_id(exported.files[0].id, "file")
+
+ file_agent = agent.files_agents[0]
+ assert file_agent.agent_id == agent.id
+ assert file_agent.file_id == exported.files[0].id
+ assert file_agent.source_id == exported.sources[0].id
+
+ async def test_multiple_files_per_source(self, server, default_user, agent_file_manager):
+ """Test export with multiple files from the same source"""
+ source = await create_test_source(server, "multi-file-source", default_user)
+ file1 = await create_test_file(server, "file1.txt", source.id, default_user)
+ file2 = await create_test_file(server, "file2.txt", source.id, default_user)
+
+ agent = await create_test_agent_with_files(server, "multi-file-agent", default_user, [(source.id, file1.id), (source.id, file2.id)])
+
+ exported = await agent_file_manager.export([agent.id], actor=default_user)
+
+ assert len(exported.agents) == 1
+ assert len(exported.sources) == 1
+ assert len(exported.files) == 2
+
+ agent = exported.agents[0]
+ assert len(agent.files_agents) == 2
+
+ source_id = exported.sources[0].id
+ for file_schema in exported.files:
+ assert file_schema.source_id == source_id
+
+ file_ids = {f.id for f in exported.files}
+ for file_agent in agent.files_agents:
+ assert file_agent.file_id in file_ids
+ assert file_agent.source_id == source_id
+
+ async def test_multiple_sources_export(self, server, default_user, agent_file_manager):
+ """Test export with files from multiple sources"""
+ source1 = await create_test_source(server, "source-1", default_user)
+ source2 = await create_test_source(server, "source-2", default_user)
+ file1 = await create_test_file(server, "file1.txt", source1.id, default_user)
+ file2 = await create_test_file(server, "file2.txt", source2.id, default_user)
+
+ agent = await create_test_agent_with_files(
+ server, "multi-source-agent", default_user, [(source1.id, file1.id), (source2.id, file2.id)]
+ )
+
+ exported = await agent_file_manager.export([agent.id], actor=default_user)
+
+ assert len(exported.agents) == 1
+ assert len(exported.sources) == 2
+ assert len(exported.files) == 2
+
+ source_ids = {s.id for s in exported.sources}
+ for file_schema in exported.files:
+ assert file_schema.source_id in source_ids
+
+ async def test_cross_agent_file_deduplication(self, server, default_user, agent_file_manager):
+ """Test that files shared across agents are deduplicated in export"""
+ source = await create_test_source(server, "shared-source", default_user)
+ shared_file = await create_test_file(server, "shared.txt", source.id, default_user)
+
+ agent1 = await create_test_agent_with_files(server, "agent-1", default_user, [(source.id, shared_file.id)])
+ agent2 = await create_test_agent_with_files(server, "agent-2", default_user, [(source.id, shared_file.id)])
+
+ exported = await agent_file_manager.export([agent1.id, agent2.id], actor=default_user)
+
+ assert len(exported.agents) == 2
+ assert len(exported.sources) == 1
+ assert len(exported.files) == 1
+
+ file_id = exported.files[0].id
+ source_id = exported.sources[0].id
+
+ for agent in exported.agents:
+ assert len(agent.files_agents) == 1
+ file_agent = agent.files_agents[0]
+ assert file_agent.file_id == file_id
+ assert file_agent.source_id == source_id
+
+ async def test_file_agent_relationship_preservation(self, server, default_user, agent_file_manager):
+ """Test that file-agent relationship details are preserved"""
+ source = await create_test_source(server, "test-source", default_user)
+ file = await create_test_file(server, "test.txt", source.id, default_user)
+
+ agent = await create_test_agent_with_files(server, "test-agent", default_user, [(source.id, file.id)])
+
+ exported = await agent_file_manager.export([agent.id], actor=default_user)
+
+ agent = exported.agents[0]
+ file_agent = agent.files_agents[0]
+
+ assert file_agent.file_name == file.file_name
+ assert file_agent.is_open is True
+ assert hasattr(file_agent, "last_accessed_at")
+
+ async def test_id_remapping_consistency(self, server, default_user, agent_file_manager):
+ """Test that ID remapping is consistent across all references"""
+ source = await create_test_source(server, "consistency-source", default_user)
+ file = await create_test_file(server, "consistency.txt", source.id, default_user)
+ agent = await create_test_agent_with_files(server, "consistency-agent", default_user, [(source.id, file.id)])
+
+ exported = await agent_file_manager.export([agent.id], actor=default_user)
+
+ agent_schema = exported.agents[0]
+ source_schema = exported.sources[0]
+ file_schema = exported.files[0]
+ file_agent = agent_schema.files_agents[0]
+
+ assert file_schema.source_id == source_schema.id
+ assert file_agent.agent_id == agent_schema.id
+ assert file_agent.file_id == file_schema.id
+ assert file_agent.source_id == source_schema.id
+
+ async def test_empty_file_relationships(self, server, default_user, agent_file_manager):
+ """Test export of agent with no file relationships"""
+ agent_create = CreateAgent(
+ name="no-files-agent",
+ llm_config=LLMConfig.default_config("gpt-4o-mini"),
+ embedding_config=EmbeddingConfig.default_config(provider="openai"),
+ )
+ agent = await server.agent_manager.create_agent_async(agent_create, actor=default_user)
+
+ exported = await agent_file_manager.export([agent.id], actor=default_user)
+
+ assert len(exported.agents) == 1
+ assert len(exported.sources) == 0
+ assert len(exported.files) == 0
+
+ agent_schema = exported.agents[0]
+ assert len(agent_schema.files_agents) == 0
+
+ async def test_file_content_inclusion_in_export(self, default_user, agent_file_manager, agent_with_files):
+ """Test that file content is included in export"""
+ agent_id, source_id, file_id = agent_with_files
+
+ exported = await agent_file_manager.export([agent_id], actor=default_user)
+
+ file_schema = exported.files[0]
+ assert hasattr(file_schema, "content") or file_schema.content is not None
+
+
class TestAgentFileExport:
"""Tests for agent file export functionality."""
async def test_basic_export(self, agent_file_manager, test_agent, default_user):
"""Test basic agent export functionality."""
- # Export the agent
agent_file = await agent_file_manager.export([test_agent.id], default_user)
- # Validate the structure
assert isinstance(agent_file, AgentFileSchema)
assert len(agent_file.agents) == 1
assert len(agent_file.tools) > 0 # Should include base tools + weather tool
assert len(agent_file.blocks) > 0 # Should include memory blocks + test block
- # Validate revision_id is automatically set in metadata
assert agent_file.metadata.get("revision_id") is not None
assert agent_file.metadata.get("revision_id") != "unknown"
assert len(agent_file.metadata.get("revision_id")) > 0
- # Validate ID formats
assert validate_id_format(agent_file)
- # Check agent data
exported_agent = agent_file.agents[0]
assert exported_agent.name == test_agent.name
assert exported_agent.system == test_agent.system
@@ -542,7 +852,6 @@ class TestAgentFileExport:
async def test_export_multiple_agents(self, server, agent_file_manager, test_agent, default_user, weather_tool):
"""Test exporting multiple agents."""
- # Create a second agent
create_agent_request = CreateAgent(
name="second_test_agent",
system="Second test agent",
@@ -559,29 +868,23 @@ class TestAgentFileExport:
actor=default_user,
)
- # Export both agents
agent_file = await agent_file_manager.export([test_agent.id, second_agent.id], default_user)
- # Validate
assert len(agent_file.agents) == 2
assert validate_id_format(agent_file)
- # Check that agents have different IDs but tools are deduplicated
agent_ids = {agent.id for agent in agent_file.agents}
assert len(agent_ids) == 2
async def test_export_id_remapping(self, agent_file_manager, test_agent, default_user):
"""Test that IDs are properly remapped during export."""
- # Export the agent
agent_file = await agent_file_manager.export([test_agent.id], default_user)
exported_agent = agent_file.agents[0]
- # Verify agent ID is remapped
assert exported_agent.id == "agent-0"
assert exported_agent.id != test_agent.id
- # Verify tool/block IDs are remapped
if exported_agent.tool_ids:
for tool_id in exported_agent.tool_ids:
assert tool_id.startswith("tool-")
@@ -590,25 +893,21 @@ class TestAgentFileExport:
for block_id in exported_agent.block_ids:
assert block_id.startswith("block-")
- # Verify message IDs are remapped and in-context references are consistent
message_ids = {msg.id for msg in exported_agent.messages}
for in_context_id in exported_agent.in_context_message_ids:
assert in_context_id in message_ids, f"In-context message ID {in_context_id} not found in messages"
async def test_message_agent_id_remapping(self, agent_file_manager, test_agent, default_user):
"""Test that message.agent_id is properly remapped during export."""
- # Export the agent
agent_file = await agent_file_manager.export([test_agent.id], default_user)
exported_agent = agent_file.agents[0]
- # Verify all messages have the remapped agent_id matching the exported agent
for message in exported_agent.messages:
assert (
message.agent_id == exported_agent.id
), f"Message {message.id} has agent_id {message.agent_id}, expected {exported_agent.id}"
- # Verify agent_id is the remapped file ID format
assert exported_agent.id == "agent-0"
assert exported_agent.id != test_agent.id
@@ -627,21 +926,16 @@ class TestAgentFileExport:
async def test_revision_id_automatic_setting(self, agent_file_manager, test_agent, default_user):
"""Test that revision_id is automatically set to the latest alembic revision."""
- # Export the agent
agent_file = await agent_file_manager.export([test_agent.id], default_user)
- # Get the expected revision ID from the function
from letta.utils import get_latest_alembic_revision
expected_revision = await get_latest_alembic_revision()
- # Validate that the revision_id matches the latest alembic revision
assert agent_file.metadata.get("revision_id") == expected_revision
- # Validate that it's not the fallback "unknown" value
assert agent_file.metadata.get("revision_id") != "unknown"
- # Validate that it looks like a valid revision ID (12 hex characters)
assert len(agent_file.metadata.get("revision_id")) == 12
assert all(c in "0123456789abcdef" for c in agent_file.metadata.get("revision_id"))
@@ -651,52 +945,41 @@ class TestAgentFileImport:
async def test_basic_import(self, agent_file_manager, test_agent, default_user, other_user):
"""Test basic agent import functionality."""
- # Export the agent
agent_file = await agent_file_manager.export([test_agent.id], default_user)
- # Import the agent
result = await agent_file_manager.import_file(agent_file, other_user)
- # Validate import result
assert result.success
assert result.imported_count > 0
assert len(result.id_mappings) > 0
- # Verify new entities were created (not existing ones reused)
for file_id, db_id in result.id_mappings.items():
if file_id.startswith("agent-"):
assert db_id != test_agent.id # New agent should have different ID
async def test_import_preserves_data(self, server, agent_file_manager, test_agent, default_user, other_user):
"""Test that import preserves all important data."""
- # Export the agent
agent_file = await agent_file_manager.export([test_agent.id], default_user)
- # Import the agent
result = await agent_file_manager.import_file(agent_file, other_user)
- # Get the imported agent
imported_agent_id = next(db_id for file_id, db_id in result.id_mappings.items() if file_id == "agent-0")
imported_agent = server.agent_manager.get_agent_by_id(imported_agent_id, other_user)
- # Compare key fields
assert imported_agent.name == test_agent.name
assert imported_agent.system == test_agent.system
assert imported_agent.description == test_agent.description
assert imported_agent.metadata == test_agent.metadata
assert imported_agent.tags == test_agent.tags
- # Check that tools and blocks were imported
assert len(imported_agent.tools) == len(test_agent.tools)
assert len(imported_agent.memory.blocks) == len(test_agent.memory.blocks)
- # Check that messages were imported
original_messages = server.message_manager.list_messages_for_agent(test_agent.id, default_user)
imported_messages = server.message_manager.list_messages_for_agent(imported_agent_id, other_user)
assert len(imported_messages) == len(original_messages)
- # Verify message content is preserved
for orig_msg, imp_msg in zip(original_messages, imported_messages):
assert orig_msg.role == imp_msg.role
assert orig_msg.content == imp_msg.content
@@ -704,20 +987,15 @@ class TestAgentFileImport:
async def test_import_message_context_preservation(self, server, agent_file_manager, test_agent, default_user, other_user):
"""Test that in-context message references are preserved during import."""
- # Export the agent
agent_file = await agent_file_manager.export([test_agent.id], default_user)
- # Import the agent
result = await agent_file_manager.import_file(agent_file, other_user)
- # Get the imported agent
imported_agent_id = next(db_id for file_id, db_id in result.id_mappings.items() if file_id == "agent-0")
imported_agent = server.agent_manager.get_agent_by_id(imported_agent_id, other_user)
- # Check that in-context message count is preserved
assert len(imported_agent.message_ids) == len(test_agent.message_ids)
- # Verify all in-context messages exist
imported_messages = server.message_manager.list_messages_for_agent(imported_agent_id, other_user)
imported_message_ids = {msg.id for msg in imported_messages}
@@ -726,13 +1004,10 @@ class TestAgentFileImport:
async def test_dry_run_import(self, agent_file_manager, test_agent, default_user, other_user):
"""Test dry run import validation."""
- # Export the agent
agent_file = await agent_file_manager.export([test_agent.id], default_user)
- # Dry run import
result = await agent_file_manager.import_file(agent_file, other_user, dry_run=True)
- # Validate dry run result
assert result.success
assert result.imported_count == 0 # No actual imports in dry run
assert len(result.id_mappings) == 0
@@ -740,12 +1015,10 @@ class TestAgentFileImport:
async def test_import_validation_errors(self, agent_file_manager, other_user):
"""Test import validation catches errors."""
- # Get current revision for test
from letta.utils import get_latest_alembic_revision
current_revision = await get_latest_alembic_revision()
- # Create invalid agent file with duplicate IDs
invalid_agent_file = AgentFileSchema(
metadata={"revision_id": current_revision},
agents=[
@@ -759,29 +1032,101 @@ class TestAgentFileImport:
tools=[],
)
- # Import should fail validation
with pytest.raises(AgentFileImportError):
await agent_file_manager.import_file(invalid_agent_file, other_user)
+class TestAgentFileImportWithProcessing:
+ """Tests for agent file import with file processing (chunking/embedding)."""
+
+ async def test_import_with_file_processing(self, server, agent_file_manager, default_user, other_user):
+ """Test that import processes files for chunking and embedding."""
+ source = await create_test_source(server, "processing-source", default_user)
+ file_content = "This is test content for processing. It should be chunked and embedded during import."
+ file_metadata = await create_test_file(server, "process.txt", source.id, default_user, content=file_content)
+
+ agent = await create_test_agent_with_files(server, "processing-agent", default_user, [(source.id, file_metadata.id)])
+
+ exported = await agent_file_manager.export([agent.id], default_user)
+
+ result = await agent_file_manager.import_file(exported, other_user)
+
+ assert result.success
+ assert result.imported_count > 0
+
+ imported_file_id = next(db_id for file_id, db_id in result.id_mappings.items() if file_id.startswith("file-"))
+
+ imported_file = await server.file_manager.get_file_by_id(imported_file_id, other_user)
+ assert imported_file.processing_status.value == "completed"
+
+ async def test_import_passage_creation(self, server, agent_file_manager, default_user, other_user):
+ """Test that import creates passages for file content."""
+ source = await create_test_source(server, "passage-source", default_user)
+ file_content = "This content should create passages. Each sentence should be chunked separately."
+ file_metadata = await create_test_file(server, "passages.txt", source.id, default_user, content=file_content)
+
+ agent = await create_test_agent_with_files(server, "passage-agent", default_user, [(source.id, file_metadata.id)])
+
+ exported = await agent_file_manager.export([agent.id], default_user)
+
+ result = await agent_file_manager.import_file(exported, other_user)
+
+ imported_file_id = next(db_id for file_id, db_id in result.id_mappings.items() if file_id.startswith("file-"))
+
+ passages = await server.passage_manager.list_passages_by_file_id_async(imported_file_id, other_user)
+ assert len(passages) > 0
+
+ for passage in passages:
+ assert passage.embedding is not None
+ assert len(passage.embedding) > 0
+
+ async def test_import_file_status_updates(self, server, agent_file_manager, default_user, other_user):
+ """Test that file processing status is updated correctly during import."""
+ source = await create_test_source(server, "status-source", default_user)
+ file_metadata = await create_test_file(server, "status.txt", source.id, default_user)
+
+ agent = await create_test_agent_with_files(server, "status-agent", default_user, [(source.id, file_metadata.id)])
+
+ exported = await agent_file_manager.export([agent.id], default_user)
+
+ result = await agent_file_manager.import_file(exported, other_user)
+
+ imported_file_id = next(db_id for file_id, db_id in result.id_mappings.items() if file_id.startswith("file-"))
+ imported_file = await server.file_manager.get_file_by_id(imported_file_id, other_user)
+
+ assert imported_file.processing_status.value == "completed"
+ assert imported_file.total_chunks is None
+ assert imported_file.chunks_embedded is None
+
+ async def test_import_multiple_files_processing(self, server, agent_file_manager, default_user, other_user):
+ """Test import processes multiple files efficiently."""
+ source = await create_test_source(server, "multi-source", default_user)
+ file1 = await create_test_file(server, "file1.txt", source.id, default_user)
+ file2 = await create_test_file(server, "file2.txt", source.id, default_user)
+
+ agent = await create_test_agent_with_files(server, "multi-agent", default_user, [(source.id, file1.id), (source.id, file2.id)])
+
+ exported = await agent_file_manager.export([agent.id], default_user)
+
+ result = await agent_file_manager.import_file(exported, other_user)
+
+ imported_file_ids = [db_id for file_id, db_id in result.id_mappings.items() if file_id.startswith("file-")]
+ assert len(imported_file_ids) == 2
+
+ for file_id in imported_file_ids:
+ imported_file = await server.file_manager.get_file_by_id(file_id, other_user)
+ assert imported_file.processing_status.value == "completed"
+
+
class TestAgentFileRoundTrip:
"""Tests for complete export -> import -> export cycles."""
async def test_roundtrip_consistency(self, server, agent_file_manager, test_agent, default_user, other_user):
"""Test that export -> import -> export produces consistent results."""
- # First export
original_export = await agent_file_manager.export([test_agent.id], default_user)
-
- # Import
result = await agent_file_manager.import_file(original_export, other_user)
-
- # Get imported agent ID
imported_agent_id = next(db_id for file_id, db_id in result.id_mappings.items() if file_id == "agent-0")
-
- # Second export
second_export = await agent_file_manager.export([imported_agent_id], other_user)
-
- # Compare exports (should be logically equivalent)
assert compare_agent_files(original_export, second_export)
async def test_multiple_roundtrips(self, server, agent_file_manager, test_agent, default_user, other_user):
@@ -790,18 +1135,14 @@ class TestAgentFileRoundTrip:
current_user = default_user
for i in range(3):
- # Export
agent_file = await agent_file_manager.export([current_agent_id], current_user)
- # Import to other user
target_user = other_user if current_user == default_user else default_user
result = await agent_file_manager.import_file(agent_file, target_user)
- # Update for next iteration
current_agent_id = next(db_id for file_id, db_id in result.id_mappings.items() if file_id == "agent-0")
current_user = target_user
- # Verify the agent still works
imported_agent = server.agent_manager.get_agent_by_id(current_agent_id, current_user)
assert imported_agent.name == test_agent.name