Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

chore: create file manager (#2693)

cthomasc300e58db973
diff --git a/letta/data_sources/connectors.py b/letta/data_sources/connectors.py
index ce12d5733..3248e83ca 100644
--- a/letta/data_sources/connectors.py
+++ b/letta/data_sources/connectors.py
@@ -8,8 +8,8 @@ from letta.embeddings import embedding_model
from letta.schemas.file import FileMetadata
from letta.schemas.passage import Passage
from letta.schemas.source import Source
+from letta.services.file_manager import FileManager
from letta.services.passage_manager import PassageManager
-from letta.services.source_manager import SourceManager
class DataConnector:
@@ -38,9 +38,7 @@ class DataConnector:
"""
-async def load_data(
- connector: DataConnector, source: Source, passage_manager: PassageManager, source_manager: SourceManager, actor: "User"
-):
+async def load_data(connector: DataConnector, source: Source, passage_manager: PassageManager, file_manager: FileManager, actor: "User"):
from letta.llm_api.llm_client import LLMClient
from letta.schemas.embedding_config import EmbeddingConfig
@@ -94,7 +92,7 @@ async def load_data(
for file_metadata in connector.find_files(source):
file_count += 1
- await source_manager.create_file(file_metadata, actor)
+ await file_manager.create_file(file_metadata, actor)
# generate passages
for passage_text, passage_metadata in connector.generate_passages(file_metadata, chunk_size=embedding_config.embedding_chunk_size):
diff --git a/letta/server/rest_api/routers/v1/agents.py b/letta/server/rest_api/routers/v1/agents.py
index b7dfeb9a3..7a6d71633 100644
--- a/letta/server/rest_api/routers/v1/agents.py
+++ b/letta/server/rest_api/routers/v1/agents.py
@@ -316,7 +316,7 @@ async def attach_source(
# Check if the agent is missing any files tools
agent_state = await server.agent_manager.attach_missing_files_tools_async(agent_state=agent_state, actor=actor)
- files = await server.source_manager.list_files(source_id, actor, include_content=True)
+ files = await server.file_manager.list_files(source_id, actor, include_content=True)
texts = []
file_ids = []
file_names = []
@@ -354,7 +354,7 @@ async def detach_source(
if not agent_state.sources:
agent_state = await server.agent_manager.detach_all_files_tools_async(agent_state=agent_state, actor=actor)
- files = await server.source_manager.list_files(source_id, actor)
+ files = await server.file_manager.list_files(source_id, actor)
file_ids = [f.id for f in files]
await server.remove_files_from_context_window(agent_state=agent_state, file_ids=file_ids, actor=actor)
diff --git a/letta/server/rest_api/routers/v1/sources.py b/letta/server/rest_api/routers/v1/sources.py
index 2cdf9090f..0ff504f9f 100644
--- a/letta/server/rest_api/routers/v1/sources.py
+++ b/letta/server/rest_api/routers/v1/sources.py
@@ -154,7 +154,7 @@ async def delete_source(
actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)
source = await server.source_manager.get_source_by_id(source_id=source_id, actor=actor)
agent_states = await server.source_manager.list_attached_agents(source_id=source_id, actor=actor)
- files = await server.source_manager.list_files(source_id, actor)
+ files = await server.file_manager.list_files(source_id, actor)
file_ids = [f.id for f in files]
for agent_state in agent_states:
@@ -294,7 +294,7 @@ async def list_source_files(
List paginated files associated with a data source.
"""
actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)
- return await server.source_manager.list_files(
+ return await server.file_manager.list_files(
source_id=source_id,
limit=limit,
after=after,
@@ -317,7 +317,7 @@ async def delete_file_from_source(
"""
actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)
- deleted_file = await server.source_manager.delete_file(file_id=file_id, actor=actor)
+ deleted_file = await server.file_manager.delete_file(file_id=file_id, actor=actor)
await server.remove_file_from_context_windows(source_id=source_id, file_id=deleted_file.id, actor=actor)
diff --git a/letta/server/server.py b/letta/server/server.py
index 1ee944d3c..400505f13 100644
--- a/letta/server/server.py
+++ b/letta/server/server.py
@@ -80,6 +80,7 @@ from letta.server.rest_api.interface import StreamingServerInterface
from letta.server.rest_api.utils import sse_async_generator
from letta.services.agent_manager import AgentManager
from letta.services.block_manager import BlockManager
+from letta.services.file_manager import FileManager
from letta.services.files_agents_manager import FileAgentManager
from letta.services.group_manager import GroupManager
from letta.services.helpers.tool_execution_helper import prepare_local_sandbox
@@ -219,6 +220,7 @@ class SyncServer(Server):
self.batch_manager = LLMBatchManager()
self.telemetry_manager = TelemetryManager()
self.file_agent_manager = FileAgentManager()
+ self.file_manager = FileManager()
# A resusable httpx client
timeout = httpx.Timeout(connect=10.0, read=20.0, write=10.0, pool=10.0)
@@ -1507,7 +1509,7 @@ class SyncServer(Server):
raise ValueError(f"Data source {source_name} does not exist for user {user_id}")
# load data into the document store
- passage_count, document_count = await load_data(connector, source, self.passage_manager, self.source_manager, actor=actor)
+ passage_count, document_count = await load_data(connector, source, self.passage_manager, self.file_manager, actor=actor)
return passage_count, document_count
def list_data_source_passages(self, user_id: str, source_id: str) -> List[Passage]:
diff --git a/letta/services/file_manager.py b/letta/services/file_manager.py
new file mode 100644
index 000000000..ee43dd54b
--- /dev/null
+++ b/letta/services/file_manager.py
@@ -0,0 +1,219 @@
+from datetime import datetime
+from typing import List, Optional
+
+from sqlalchemy import select, update
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import selectinload
+
+from letta.orm.errors import NoResultFound
+from letta.orm.file import FileContent as FileContentModel
+from letta.orm.file import FileMetadata as FileMetadataModel
+from letta.orm.sqlalchemy_base import AccessType
+from letta.otel.tracing import trace_method
+from letta.schemas.enums import FileProcessingStatus
+from letta.schemas.file import FileMetadata as PydanticFileMetadata
+from letta.schemas.user import User as PydanticUser
+from letta.server.db import db_registry
+from letta.utils import enforce_types
+
+
+class FileManager:
+ """Manager class to handle business logic related to files."""
+
+ @enforce_types
+ @trace_method
+ async def create_file(
+ self,
+ file_metadata: PydanticFileMetadata,
+ actor: PydanticUser,
+ *,
+ text: Optional[str] = None,
+ ) -> PydanticFileMetadata:
+
+ # short-circuit if it already exists
+ existing = await self.get_file_by_id(file_metadata.id, actor=actor)
+ if existing:
+ return existing
+
+ async with db_registry.async_session() as session:
+ try:
+ file_metadata.organization_id = actor.organization_id
+ file_orm = FileMetadataModel(**file_metadata.model_dump(to_orm=True, exclude_none=True))
+ await file_orm.create_async(session, actor=actor, no_commit=True)
+
+ if text is not None:
+ content_orm = FileContentModel(file_id=file_orm.id, text=text)
+ await content_orm.create_async(session, actor=actor, no_commit=True)
+
+ await session.commit()
+ await session.refresh(file_orm)
+ return await file_orm.to_pydantic_async()
+
+ except IntegrityError:
+ await session.rollback()
+ return await self.get_file_by_id(file_metadata.id, actor=actor)
+
+ # TODO: We make actor optional for now, but should most likely be enforced due to security reasons
+ @enforce_types
+ @trace_method
+ async def get_file_by_id(
+ self,
+ file_id: str,
+ actor: Optional[PydanticUser] = None,
+ *,
+ include_content: bool = False,
+ ) -> Optional[PydanticFileMetadata]:
+ """Retrieve a file by its ID.
+
+ If `include_content=True`, the FileContent relationship is eagerly
+ loaded so `to_pydantic(include_content=True)` never triggers a
+ lazy SELECT (avoids MissingGreenlet).
+ """
+ async with db_registry.async_session() as session:
+ try:
+ if include_content:
+ # explicit eager load
+ query = (
+ select(FileMetadataModel).where(FileMetadataModel.id == file_id).options(selectinload(FileMetadataModel.content))
+ )
+ # apply org-scoping if actor provided
+ if actor:
+ query = FileMetadataModel.apply_access_predicate(
+ query,
+ actor,
+ access=["read"],
+ access_type=AccessType.ORGANIZATION,
+ )
+
+ result = await session.execute(query)
+ file_orm = result.scalar_one()
+ else:
+ # fast path (metadata only)
+ file_orm = await FileMetadataModel.read_async(
+ db_session=session,
+ identifier=file_id,
+ actor=actor,
+ )
+
+ return await file_orm.to_pydantic_async(include_content=include_content)
+
+ except NoResultFound:
+ return None
+
+ @enforce_types
+ @trace_method
+ async def update_file_status(
+ self,
+ *,
+ file_id: str,
+ actor: PydanticUser,
+ processing_status: Optional[FileProcessingStatus] = None,
+ error_message: Optional[str] = None,
+ ) -> PydanticFileMetadata:
+ """
+ Update processing_status and/or error_message on a FileMetadata row.
+
+ * 1st round-trip → UPDATE
+ * 2nd round-trip → SELECT fresh row (same as read_async)
+ """
+
+ if processing_status is None and error_message is None:
+ raise ValueError("Nothing to update")
+
+ values: dict[str, object] = {"updated_at": datetime.utcnow()}
+ if processing_status is not None:
+ values["processing_status"] = processing_status
+ if error_message is not None:
+ values["error_message"] = error_message
+
+ async with db_registry.async_session() as session:
+ # Fast in-place update – no ORM hydration
+ stmt = (
+ update(FileMetadataModel)
+ .where(
+ FileMetadataModel.id == file_id,
+ FileMetadataModel.organization_id == actor.organization_id,
+ )
+ .values(**values)
+ )
+ await session.execute(stmt)
+ await session.commit()
+
+ # Reload via normal accessor so we return a fully-attached object
+ file_orm = await FileMetadataModel.read_async(
+ db_session=session,
+ identifier=file_id,
+ actor=actor,
+ )
+ return await file_orm.to_pydantic_async()
+
+ @enforce_types
+ @trace_method
+ async def upsert_file_content(
+ self,
+ *,
+ file_id: str,
+ text: str,
+ actor: PydanticUser,
+ ) -> PydanticFileMetadata:
+ async with db_registry.async_session() as session:
+ await FileMetadataModel.read_async(session, file_id, actor)
+
+ dialect_name = session.bind.dialect.name
+
+ if dialect_name == "postgresql":
+ stmt = (
+ pg_insert(FileContentModel)
+ .values(file_id=file_id, text=text)
+ .on_conflict_do_update(
+ index_elements=[FileContentModel.file_id],
+ set_={"text": text},
+ )
+ )
+ await session.execute(stmt)
+ else:
+ # Emulate upsert for SQLite and others
+ stmt = select(FileContentModel).where(FileContentModel.file_id == file_id)
+ result = await session.execute(stmt)
+ existing = result.scalar_one_or_none()
+
+ if existing:
+ await session.execute(update(FileContentModel).where(FileContentModel.file_id == file_id).values(text=text))
+ else:
+ session.add(FileContentModel(file_id=file_id, text=text))
+
+ await session.commit()
+
+ # Reload with content
+ query = select(FileMetadataModel).options(selectinload(FileMetadataModel.content)).where(FileMetadataModel.id == file_id)
+ result = await session.execute(query)
+ return await result.scalar_one().to_pydantic_async(include_content=True)
+
+ @enforce_types
+ @trace_method
+ async def list_files(
+ self, source_id: str, actor: PydanticUser, after: Optional[str] = None, limit: Optional[int] = 50, include_content: bool = False
+ ) -> List[PydanticFileMetadata]:
+ """List all files with optional pagination."""
+ async with db_registry.async_session() as session:
+ options = [selectinload(FileMetadataModel.content)] if include_content else None
+
+ files = await FileMetadataModel.list_async(
+ db_session=session,
+ after=after,
+ limit=limit,
+ organization_id=actor.organization_id,
+ source_id=source_id,
+ query_options=options,
+ )
+ return [await file.to_pydantic_async(include_content=include_content) for file in files]
+
+ @enforce_types
+ @trace_method
+ async def delete_file(self, file_id: str, actor: PydanticUser) -> PydanticFileMetadata:
+ """Delete a file by its ID."""
+ async with db_registry.async_session() as session:
+ file = await FileMetadataModel.read_async(db_session=session, identifier=file_id)
+ await file.hard_delete_async(db_session=session, actor=actor)
+ return await file.to_pydantic_async()
diff --git a/letta/services/file_processor/file_processor.py b/letta/services/file_processor/file_processor.py
index c82786cc7..20e2bf50e 100644
--- a/letta/services/file_processor/file_processor.py
+++ b/letta/services/file_processor/file_processor.py
@@ -11,6 +11,7 @@ from letta.schemas.job import Job, JobUpdate
from letta.schemas.passage import Passage
from letta.schemas.user import User
from letta.server.server import SyncServer
+from letta.services.file_manager import FileManager
from letta.services.file_processor.chunker.line_chunker import LineChunker
from letta.services.file_processor.chunker.llama_index_chunker import LlamaIndexChunker
from letta.services.file_processor.embedder.openai_embedder import OpenAIEmbedder
@@ -38,6 +39,7 @@ class FileProcessor:
self.line_chunker = LineChunker()
self.embedder = embedder
self.max_file_size = max_file_size
+ self.file_manager = FileManager()
self.source_manager = SourceManager()
self.passage_manager = PassageManager()
self.job_manager = JobManager()
@@ -58,7 +60,7 @@ class FileProcessor:
# Create file as early as possible with no content
file_metadata.processing_status = FileProcessingStatus.PARSING # Parsing now
- file_metadata = await self.source_manager.create_file(file_metadata, self.actor)
+ file_metadata = await self.file_manager.create_file(file_metadata, self.actor)
try:
# Ensure we're working with bytes
@@ -73,10 +75,8 @@ class FileProcessor:
# update file with raw text
raw_markdown_text = "".join([page.markdown for page in ocr_response.pages])
- file_metadata = await self.source_manager.upsert_file_content(
- file_id=file_metadata.id, text=raw_markdown_text, actor=self.actor
- )
- file_metadata = await self.source_manager.update_file_status(
+ file_metadata = await self.file_manager.upsert_file_content(file_id=file_metadata.id, text=raw_markdown_text, actor=self.actor)
+ file_metadata = await self.file_manager.update_file_status(
file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.EMBEDDING
)
@@ -123,7 +123,7 @@ class FileProcessor:
job.metadata["num_passages"] = len(all_passages)
await self.job_manager.update_job_by_id_async(job_id=job.id, job_update=JobUpdate(**job.model_dump()), actor=self.actor)
- await self.source_manager.update_file_status(
+ await self.file_manager.update_file_status(
file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.COMPLETED
)
@@ -138,7 +138,7 @@ class FileProcessor:
job.metadata["error"] = str(e)
await self.job_manager.update_job_by_id_async(job_id=job.id, job_update=JobUpdate(**job.model_dump()), actor=self.actor)
- await self.source_manager.update_file_status(
+ await self.file_manager.update_file_status(
file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.ERROR, error_message=str(e)
)
diff --git a/letta/services/source_manager.py b/letta/services/source_manager.py
index 614c0c31c..88aa4c6d3 100644
--- a/letta/services/source_manager.py
+++ b/letta/services/source_manager.py
@@ -1,21 +1,10 @@
import asyncio
-from datetime import datetime
from typing import List, Optional
-from sqlalchemy import select, update
-from sqlalchemy.dialects.postgresql import insert as pg_insert
-from sqlalchemy.exc import IntegrityError
-from sqlalchemy.orm import selectinload
-
from letta.orm.errors import NoResultFound
-from letta.orm.file import FileContent as FileContentModel
-from letta.orm.file import FileMetadata as FileMetadataModel
from letta.orm.source import Source as SourceModel
-from letta.orm.sqlalchemy_base import AccessType
from letta.otel.tracing import trace_method
from letta.schemas.agent import AgentState as PydanticAgentState
-from letta.schemas.enums import FileProcessingStatus
-from letta.schemas.file import FileMetadata as PydanticFileMetadata
from letta.schemas.source import Source as PydanticSource
from letta.schemas.source import SourceUpdate
from letta.schemas.user import User as PydanticUser
@@ -148,200 +137,3 @@ class SourceManager:
return None
else:
return sources[0].to_pydantic()
-
- @enforce_types
- @trace_method
- async def create_file(
- self,
- file_metadata: PydanticFileMetadata,
- actor: PydanticUser,
- *,
- text: Optional[str] = None,
- ) -> PydanticFileMetadata:
-
- # short-circuit if it already exists
- existing = await self.get_file_by_id(file_metadata.id, actor=actor)
- if existing:
- return existing
-
- async with db_registry.async_session() as session:
- try:
- file_metadata.organization_id = actor.organization_id
- file_orm = FileMetadataModel(**file_metadata.model_dump(to_orm=True, exclude_none=True))
- await file_orm.create_async(session, actor=actor, no_commit=True)
-
- if text is not None:
- content_orm = FileContentModel(file_id=file_orm.id, text=text)
- await content_orm.create_async(session, actor=actor, no_commit=True)
-
- await session.commit()
- await session.refresh(file_orm)
- return await file_orm.to_pydantic_async()
-
- except IntegrityError:
- await session.rollback()
- return await self.get_file_by_id(file_metadata.id, actor=actor)
-
- # TODO: We make actor optional for now, but should most likely be enforced due to security reasons
- @enforce_types
- @trace_method
- async def get_file_by_id(
- self,
- file_id: str,
- actor: Optional[PydanticUser] = None,
- *,
- include_content: bool = False,
- ) -> Optional[PydanticFileMetadata]:
- """Retrieve a file by its ID.
-
- If `include_content=True`, the FileContent relationship is eagerly
- loaded so `to_pydantic(include_content=True)` never triggers a
- lazy SELECT (avoids MissingGreenlet).
- """
- async with db_registry.async_session() as session:
- try:
- if include_content:
- # explicit eager load
- query = (
- select(FileMetadataModel).where(FileMetadataModel.id == file_id).options(selectinload(FileMetadataModel.content))
- )
- # apply org-scoping if actor provided
- if actor:
- query = FileMetadataModel.apply_access_predicate(
- query,
- actor,
- access=["read"],
- access_type=AccessType.ORGANIZATION,
- )
-
- result = await session.execute(query)
- file_orm = result.scalar_one()
- else:
- # fast path (metadata only)
- file_orm = await FileMetadataModel.read_async(
- db_session=session,
- identifier=file_id,
- actor=actor,
- )
-
- return await file_orm.to_pydantic_async(include_content=include_content)
-
- except NoResultFound:
- return None
-
- @enforce_types
- @trace_method
- async def update_file_status(
- self,
- *,
- file_id: str,
- actor: PydanticUser,
- processing_status: Optional[FileProcessingStatus] = None,
- error_message: Optional[str] = None,
- ) -> PydanticFileMetadata:
- """
- Update processing_status and/or error_message on a FileMetadata row.
-
- * 1st round-trip → UPDATE
- * 2nd round-trip → SELECT fresh row (same as read_async)
- """
-
- if processing_status is None and error_message is None:
- raise ValueError("Nothing to update")
-
- values: dict[str, object] = {"updated_at": datetime.utcnow()}
- if processing_status is not None:
- values["processing_status"] = processing_status
- if error_message is not None:
- values["error_message"] = error_message
-
- async with db_registry.async_session() as session:
- # Fast in-place update – no ORM hydration
- stmt = (
- update(FileMetadataModel)
- .where(
- FileMetadataModel.id == file_id,
- FileMetadataModel.organization_id == actor.organization_id,
- )
- .values(**values)
- )
- await session.execute(stmt)
- await session.commit()
-
- # Reload via normal accessor so we return a fully-attached object
- file_orm = await FileMetadataModel.read_async(
- db_session=session,
- identifier=file_id,
- actor=actor,
- )
- return await file_orm.to_pydantic_async()
-
- @enforce_types
- @trace_method
- async def upsert_file_content(
- self,
- *,
- file_id: str,
- text: str,
- actor: PydanticUser,
- ) -> PydanticFileMetadata:
- async with db_registry.async_session() as session:
- await FileMetadataModel.read_async(session, file_id, actor)
-
- dialect_name = session.bind.dialect.name
-
- if dialect_name == "postgresql":
- stmt = (
- pg_insert(FileContentModel)
- .values(file_id=file_id, text=text)
- .on_conflict_do_update(
- index_elements=[FileContentModel.file_id],
- set_={"text": text},
- )
- )
- await session.execute(stmt)
- else:
- # Emulate upsert for SQLite and others
- stmt = select(FileContentModel).where(FileContentModel.file_id == file_id)
- result = await session.execute(stmt)
- existing = result.scalar_one_or_none()
-
- if existing:
- await session.execute(update(FileContentModel).where(FileContentModel.file_id == file_id).values(text=text))
- else:
- session.add(FileContentModel(file_id=file_id, text=text))
-
- await session.commit()
-
- # Reload with content
- query = select(FileMetadataModel).options(selectinload(FileMetadataModel.content)).where(FileMetadataModel.id == file_id)
- result = await session.execute(query)
- return await result.scalar_one().to_pydantic_async(include_content=True)
-
- @enforce_types
- @trace_method
- async def list_files(
- self, source_id: str, actor: PydanticUser, after: Optional[str] = None, limit: Optional[int] = 50, include_content: bool = False
- ) -> List[PydanticFileMetadata]:
- """List all files with optional pagination."""
- async with db_registry.async_session() as session:
- options = [selectinload(FileMetadataModel.content)] if include_content else None
-
- files = await FileMetadataModel.list_async(
- db_session=session,
- after=after,
- limit=limit,
- organization_id=actor.organization_id,
- source_id=source_id,
- query_options=options,
- )
- return [await file.to_pydantic_async(include_content=include_content) for file in files]
-
- @enforce_types
- @trace_method
- async def delete_file(self, file_id: str, actor: PydanticUser) -> PydanticFileMetadata:
- """Delete a file by its ID."""
- async with db_registry.async_session() as session:
- file = await FileMetadataModel.read_async(db_session=session, identifier=file_id)
- await file.hard_delete_async(db_session=session, actor=actor)
- return await file.to_pydantic_async()
diff --git a/letta/services/tool_executor/files_tool_executor.py b/letta/services/tool_executor/files_tool_executor.py
index b85fe6abc..ceed535fa 100644
--- a/letta/services/tool_executor/files_tool_executor.py
+++ b/letta/services/tool_executor/files_tool_executor.py
@@ -10,6 +10,7 @@ from letta.schemas.tool_execution_result import ToolExecutionResult
from letta.schemas.user import User
from letta.services.agent_manager import AgentManager
from letta.services.block_manager import BlockManager
+from letta.services.file_manager import FileManager
from letta.services.file_processor.chunker.line_chunker import LineChunker
from letta.services.files_agents_manager import FileAgentManager
from letta.services.message_manager import MessageManager
@@ -49,6 +50,7 @@ class LettaFileToolExecutor(ToolExecutor):
# TODO: This should be passed in to for testing purposes
self.files_agents_manager = FileAgentManager()
+ self.file_manager = FileManager()
self.source_manager = SourceManager()
self.logger = get_logger(__name__)
@@ -112,7 +114,7 @@ class LettaFileToolExecutor(ToolExecutor):
)
file_id = file_agent.file_id
- file = await self.source_manager.get_file_by_id(file_id=file_id, actor=self.actor, include_content=True)
+ file = await self.file_manager.get_file_by_id(file_id=file_id, actor=self.actor, include_content=True)
# TODO: Inefficient, maybe we can pre-compute this
# TODO: This is also not the best way to split things - would be cool to have "content aware" splitting
@@ -234,7 +236,7 @@ class LettaFileToolExecutor(ToolExecutor):
for file_agent in file_agents:
# Load file content
- file = await self.source_manager.get_file_by_id(file_id=file_agent.file_id, actor=self.actor, include_content=True)
+ file = await self.file_manager.get_file_by_id(file_id=file_agent.file_id, actor=self.actor, include_content=True)
if not file or not file.content:
files_skipped += 1
diff --git a/tests/test_managers.py b/tests/test_managers.py
index 1e17bceaf..0d12fddbe 100644
--- a/tests/test_managers.py
+++ b/tests/test_managers.py
@@ -183,7 +183,7 @@ async def other_source(server: SyncServer, default_user):
@pytest.fixture
async def default_file(server: SyncServer, default_source, default_user, default_organization):
- file = await server.source_manager.create_file(
+ file = await server.file_manager.create_file(
PydanticFileMetadata(file_name="test_file", organization_id=default_organization.id, source_id=default_source.id),
actor=default_user,
)
@@ -681,7 +681,7 @@ async def another_file(server, default_source, default_user, default_organizatio
organization_id=default_organization.id,
source_id=default_source.id,
)
- return await server.source_manager.create_file(pf, actor=default_user)
+ return await server.file_manager.create_file(pf, actor=default_user)
# ======================================================================================================================
@@ -4581,10 +4581,10 @@ async def test_get_file_by_id(server: SyncServer, default_user, default_source):
file_size=2048,
source_id=default_source.id,
)
- created_file = await server.source_manager.create_file(file_metadata=file_metadata, actor=default_user)
+ created_file = await server.file_manager.create_file(file_metadata=file_metadata, actor=default_user)
# Retrieve the file by ID
- retrieved_file = await server.source_manager.get_file_by_id(file_id=created_file.id, actor=default_user)
+ retrieved_file = await server.file_manager.get_file_by_id(file_id=created_file.id, actor=default_user)
# Assertions to verify the retrieved file matches the created one
assert retrieved_file.id == created_file.id
@@ -4605,7 +4605,7 @@ async def test_create_and_retrieve_file_with_content(server, default_user, defau
source_id=default_source.id,
)
- created = await server.source_manager.create_file(
+ created = await server.file_manager.create_file(
file_metadata=meta,
actor=default_user,
text=text_body,
@@ -4618,7 +4618,7 @@ async def test_create_and_retrieve_file_with_content(server, default_user, defau
assert await _count_file_content_rows(async_session, created.id) == 1
# -- now fetch WITH the body
- loaded = await server.source_manager.get_file_by_id(created.id, actor=default_user, include_content=True)
+ loaded = await server.file_manager.get_file_by_id(created.id, actor=default_user, include_content=True)
assert loaded.content == text_body
@@ -4631,13 +4631,13 @@ async def test_create_file_without_content(server, default_user, default_source,
file_size=123,
source_id=default_source.id,
)
- created = await server.source_manager.create_file(file_metadata=meta, actor=default_user)
+ created = await server.file_manager.create_file(file_metadata=meta, actor=default_user)
# no content row
assert await _count_file_content_rows(async_session, created.id) == 0
# include_content=True still works, returns None
- loaded = await server.source_manager.get_file_by_id(created.id, actor=default_user, include_content=True)
+ loaded = await server.file_manager.get_file_by_id(created.id, actor=default_user, include_content=True)
assert loaded.content is None
@@ -4652,7 +4652,7 @@ async def test_lazy_raise_guard(server, default_user, default_source, async_sess
file_size=len(text_body),
source_id=default_source.id,
)
- created = await server.source_manager.create_file(file_metadata=meta, actor=default_user, text=text_body)
+ created = await server.file_manager.create_file(file_metadata=meta, actor=default_user, text=text_body)
# Grab ORM instance WITHOUT selectinload(FileMetadata.content)
orm = await async_session.get(FileMetadataModel, created.id)
@@ -4664,7 +4664,7 @@ async def test_lazy_raise_guard(server, default_user, default_source, async_sess
@pytest.mark.asyncio
async def test_list_files_content_none(server, default_user, default_source):
- files = await server.source_manager.list_files(source_id=default_source.id, actor=default_user)
+ files = await server.file_manager.list_files(source_id=default_source.id, actor=default_user)
assert all(f.content is None for f in files)
@@ -4678,13 +4678,13 @@ async def test_delete_cascades_to_content(server, default_user, default_source,
file_size=len(text_body),
source_id=default_source.id,
)
- created = await server.source_manager.create_file(file_metadata=meta, actor=default_user, text=text_body)
+ created = await server.file_manager.create_file(file_metadata=meta, actor=default_user, text=text_body)
# ensure row exists first
assert await _count_file_content_rows(async_session, created.id) == 1
# delete
- await server.source_manager.delete_file(created.id, actor=default_user)
+ await server.file_manager.delete_file(created.id, actor=default_user)
# content row gone
assert await _count_file_content_rows(async_session, created.id) == 0
@@ -4694,29 +4694,27 @@ async def test_delete_cascades_to_content(server, default_user, default_source,
async def test_list_files(server: SyncServer, default_user, default_source):
"""Test listing files with pagination."""
# Create multiple files
- await server.source_manager.create_file(
+ await server.file_manager.create_file(
PydanticFileMetadata(file_name="File 1", file_path="/path/to/file1.txt", file_type="text/plain", source_id=default_source.id),
actor=default_user,
)
if USING_SQLITE:
time.sleep(CREATE_DELAY_SQLITE)
- await server.source_manager.create_file(
+ await server.file_manager.create_file(
PydanticFileMetadata(file_name="File 2", file_path="/path/to/file2.txt", file_type="text/plain", source_id=default_source.id),
actor=default_user,
)
# List files without pagination
- files = await server.source_manager.list_files(source_id=default_source.id, actor=default_user)
+ files = await server.file_manager.list_files(source_id=default_source.id, actor=default_user)
assert len(files) == 2
# List files with pagination
- paginated_files = await server.source_manager.list_files(source_id=default_source.id, actor=default_user, limit=1)
+ paginated_files = await server.file_manager.list_files(source_id=default_source.id, actor=default_user, limit=1)
assert len(paginated_files) == 1
# Ensure cursor-based pagination works
- next_page = await server.source_manager.list_files(
- source_id=default_source.id, actor=default_user, after=paginated_files[-1].id, limit=1
- )
+ next_page = await server.file_manager.list_files(source_id=default_source.id, actor=default_user, after=paginated_files[-1].id, limit=1)
assert len(next_page) == 1
assert next_page[0].file_name != paginated_files[0].file_name
@@ -4727,16 +4725,16 @@ async def test_delete_file(server: SyncServer, default_user, default_source):
file_metadata = PydanticFileMetadata(
file_name="Delete File", file_path="/path/to/delete_file.txt", file_type="text/plain", source_id=default_source.id
)
- created_file = await server.source_manager.create_file(file_metadata=file_metadata, actor=default_user)
+ created_file = await server.file_manager.create_file(file_metadata=file_metadata, actor=default_user)
# Delete the file
- deleted_file = await server.source_manager.delete_file(file_id=created_file.id, actor=default_user)
+ deleted_file = await server.file_manager.delete_file(file_id=created_file.id, actor=default_user)
# Assertions to verify deletion
assert deleted_file.id == created_file.id
# Verify that the file no longer appears in list_files
- files = await server.source_manager.list_files(source_id=default_source.id, actor=default_user)
+ files = await server.file_manager.list_files(source_id=default_source.id, actor=default_user)
assert len(files) == 0
@@ -4750,10 +4748,10 @@ async def test_update_file_status_basic(server, default_user, default_source):
file_size=100,
source_id=default_source.id,
)
- created = await server.source_manager.create_file(file_metadata=meta, actor=default_user)
+ created = await server.file_manager.create_file(file_metadata=meta, actor=default_user)
# Update status only
- updated = await server.source_manager.update_file_status(
+ updated = await server.file_manager.update_file_status(
file_id=created.id,
actor=default_user,
processing_status=FileProcessingStatus.PARSING,
@@ -4762,7 +4760,7 @@ async def test_update_file_status_basic(server, default_user, default_source):
assert updated.error_message is None
# Update both status and error message
- updated = await server.source_manager.update_file_status(
+ updated = await server.file_manager.update_file_status(
file_id=created.id,
actor=default_user,
processing_status=FileProcessingStatus.ERROR,
@@ -4782,9 +4780,9 @@ async def test_update_file_status_error_only(server, default_user, default_sourc
file_size=123,
source_id=default_source.id,
)
- created = await server.source_manager.create_file(file_metadata=meta, actor=default_user)
+ created = await server.file_manager.create_file(file_metadata=meta, actor=default_user)
- updated = await server.source_manager.update_file_status(
+ updated = await server.file_manager.update_file_status(
file_id=created.id,
actor=default_user,
error_message="Timeout while embedding",
@@ -4807,11 +4805,11 @@ async def test_upsert_file_content_basic(server: SyncServer, default_user, defau
file_size=len(initial_text),
source_id=default_source.id,
)
- created = await server.source_manager.create_file(file_metadata=meta, actor=default_user)
+ created = await server.file_manager.create_file(file_metadata=meta, actor=default_user)
assert created.content is None
# Step 2: Insert new content
- file_with_content = await server.source_manager.upsert_file_content(
+ file_with_content = await server.file_manager.upsert_file_content(
file_id=created.id,
text=initial_text,
actor=default_user,
@@ -4823,7 +4821,7 @@ async def test_upsert_file_content_basic(server: SyncServer, default_user, defau
assert count == 1
# Step 3: Update existing content
- file_with_updated_content = await server.source_manager.upsert_file_content(
+ file_with_updated_content = await server.file_manager.upsert_file_content(
file_id=created.id,
text=updated_text,
actor=default_user,
diff --git a/tests/test_sources.py b/tests/test_sources.py
index b95e289b7..959863598 100644
--- a/tests/test_sources.py
+++ b/tests/test_sources.py
@@ -58,6 +58,7 @@ def agent_state(client: LettaSDKClient):
open_file_tool = client.tools.list(name="open_file")[0]
close_file_tool = client.tools.list(name="close_file")[0]
search_files_tool = client.tools.list(name="search_files")[0]
+ grep_tool = client.tools.list(name="grep")[0]
agent_state = client.agents.create(
memory_blocks=[
@@ -68,7 +69,7 @@ def agent_state(client: LettaSDKClient):
],
model="openai/gpt-4o-mini",
embedding="openai/text-embedding-ada-002",
- tool_ids=[open_file_tool.id, close_file_tool.id, search_files_tool.id],
+ tool_ids=[open_file_tool.id, close_file_tool.id, search_files_tool.id, grep_tool.id],
)
yield agent_state