Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: move sources to folders [LET-6189] (#6199)
Sarah Woodersa466e65e6bc2
diff --git a/fern/openapi.json b/fern/openapi.jsonindex 72ceccfb5..fa46b8629 100644--- a/fern/openapi.json+++ b/fern/openapi.json@@ -12732,10 +12732,12 @@"type": "null"}],- "description": "Only list jobs associated with the source.",+ "description": "Deprecated: Use `folder_id` parameter instead. Only list jobs associated with the source.",+ "deprecated": true,"title": "Source Id"},- "description": "Only list jobs associated with the source."+ "description": "Deprecated: Use `folder_id` parameter instead. Only list jobs associated with the source.",+ "deprecated": true},{"name": "before",@@ -12894,10 +12896,12 @@"type": "null"}],- "description": "Only list jobs associated with the source.",+ "description": "Deprecated: Use `folder_id` parameter instead. Only list jobs associated with the source.",+ "deprecated": true,"title": "Source Id"},- "description": "Only list jobs associated with the source."+ "description": "Deprecated: Use `folder_id` parameter instead. Only list jobs associated with the source.",+ "deprecated": true},{"name": "before",@@ -19156,7 +19160,8 @@},"type": "array","title": "Sources",- "description": "The sources used by the agent."+ "description": "Deprecated: Use `folders` field instead. The sources used by the agent.",+ "deprecated": true},"tags": {"items": {@@ -23074,7 +23079,23 @@}],"title": "Source Ids",- "description": "The ids of the sources used by the agent."+ "description": "Deprecated: Use `folder_ids` field instead. The ids of the sources used by the agent.",+ "deprecated": true+ },+ "folder_ids": {+ "anyOf": [+ {+ "items": {+ "type": "string"+ },+ "type": "array"+ },+ {+ "type": "null"+ }+ ],+ "title": "Folder Ids",+ "description": "The ids of the folders used by the agent."},"block_ids": {"anyOf": [@@ -24843,7 +24864,8 @@"source_id": {"type": "string","title": "Source Id",- "description": "Unique identifier of the source."+ "description": "Deprecated: Use `folder_id` field instead. Unique identifier of the source.",+ "deprecated": true},"file_name": {"type": "string",@@ -25116,7 +25138,8 @@"source_id": {"type": "string","title": "Source Id",- "description": "Unique identifier of the source."+ "description": "Deprecated: Use `folder_id` field instead. Unique identifier of the source.",+ "deprecated": true},"is_open": {"type": "boolean",@@ -25164,7 +25187,8 @@"source_id": {"type": "string","title": "Source Id",- "description": "The unique identifier of the source associated with the document."+ "description": "Deprecated: Use `folder_id` field instead. The unique identifier of the source associated with the document.",+ "deprecated": true},"file_name": {"anyOf": [@@ -25353,7 +25377,8 @@"source_id": {"type": "string","title": "Source Id",- "description": "The unique identifier of the source associated with the document."+ "description": "Deprecated: Use `folder_id` field instead. The unique identifier of the source associated with the document.",+ "deprecated": true},"file_name": {"anyOf": [@@ -27239,7 +27264,23 @@}],"title": "Source Ids",- "description": "The ids of the sources used by the agent."+ "description": "Deprecated: Use `folder_ids` field instead. The ids of the sources used by the agent.",+ "deprecated": true+ },+ "folder_ids": {+ "anyOf": [+ {+ "items": {+ "type": "string"+ },+ "type": "array"+ },+ {+ "type": "null"+ }+ ],+ "title": "Folder Ids",+ "description": "The ids of the folders used by the agent."},"block_ids": {"anyOf": [@@ -31234,7 +31275,8 @@}],"title": "Source Id",- "description": "The data source of the passage."+ "description": "Deprecated: Use `folder_id` field instead. The data source of the passage.",+ "deprecated": true},"file_id": {"anyOf": [@@ -33321,12 +33363,14 @@"source_id": {"type": "string","title": "Source Id",- "description": "Unique identifier of the source"+ "description": "Deprecated: Use `folder_id` field instead. Unique identifier of the source",+ "deprecated": true},"source_name": {"type": "string","title": "Source Name",- "description": "Name of the source"+ "description": "Deprecated: Use `folder_name` field instead. Name of the source",+ "deprecated": true},"file_count": {"type": "integer",@@ -35917,7 +35961,23 @@}],"title": "Source Ids",- "description": "The ids of the sources used by the agent."+ "description": "Deprecated: Use `folder_ids` field instead. The ids of the sources used by the agent.",+ "deprecated": true+ },+ "folder_ids": {+ "anyOf": [+ {+ "items": {+ "type": "string"+ },+ "type": "array"+ },+ {+ "type": "null"+ }+ ],+ "title": "Folder Ids",+ "description": "The ids of the folders used by the agent."},"block_ids": {"anyOf": [@@ -37073,7 +37133,23 @@}],"title": "Source Ids",- "description": "The ids of the sources used by the agent."+ "description": "Deprecated: Use `folder_ids` field instead. The ids of the sources used by the agent.",+ "deprecated": true+ },+ "folder_ids": {+ "anyOf": [+ {+ "items": {+ "type": "string"+ },+ "type": "array"+ },+ {+ "type": "null"+ }+ ],+ "title": "Folder Ids",+ "description": "The ids of the folders used by the agent."},"block_ids": {"anyOf": [diff --git a/letta/schemas/agent.py b/letta/schemas/agent.pyindex b3fa1cab1..a22a46fd8 100644--- a/letta/schemas/agent.py+++ b/letta/schemas/agent.py@@ -100,7 +100,9 @@ class AgentState(OrmMetadataBase, validate_assignment=True):memory: Memory = Field(..., description="Deprecated: Use `blocks` field instead. The in-context memory of the agent.", deprecated=True)blocks: List[Block] = Field(..., description="The memory blocks used by the agent.")tools: List[Tool] = Field(..., description="The tools used by the agent.")- sources: List[Source] = Field(..., description="The sources used by the agent.")+ sources: List[Source] = Field(+ ..., description="Deprecated: Use `folders` field instead. The sources used by the agent.", deprecated=True+ )tags: List[str] = Field(..., description="The tags associated with the agent.")tool_exec_environment_variables: List[AgentEnvironmentVariable] = Field(default_factory=list,@@ -199,7 +201,10 @@ class CreateAgent(BaseModel, validate_assignment=True): ## TODO: This is a legacy field and should be removed ASAP to force `tool_ids` usagetools: Optional[List[str]] = Field(None, description="The tools used by the agent.")tool_ids: Optional[List[str]] = Field(None, description="The ids of the tools used by the agent.")- source_ids: Optional[List[str]] = Field(None, description="The ids of the sources used by the agent.")+ source_ids: Optional[List[str]] = Field(+ None, description="Deprecated: Use `folder_ids` field instead. The ids of the sources used by the agent.", deprecated=True+ )+ folder_ids: Optional[List[str]] = Field(None, description="The ids of the folders used by the agent.")block_ids: Optional[List[str]] = Field(None, description="The ids of the blocks used by the agent.")tool_rules: Optional[List[ToolRule]] = Field(None, description="The tool rules governing the agent.")tags: Optional[List[str]] = Field(None, description="The tags associated with the agent.")@@ -396,7 +401,10 @@ class InternalTemplateAgentCreate(CreateAgent):class UpdateAgent(BaseModel):name: Optional[str] = Field(None, description="The name of the agent.")tool_ids: Optional[List[str]] = Field(None, description="The ids of the tools used by the agent.")- source_ids: Optional[List[str]] = Field(None, description="The ids of the sources used by the agent.")+ source_ids: Optional[List[str]] = Field(+ None, description="Deprecated: Use `folder_ids` field instead. The ids of the sources used by the agent.", deprecated=True+ )+ folder_ids: Optional[List[str]] = Field(None, description="The ids of the folders used by the agent.")block_ids: Optional[List[str]] = Field(None, description="The ids of the blocks used by the agent.")tags: Optional[List[str]] = Field(None, description="The tags associated with the agent.")system: Optional[str] = Field(None, description="The system prompt used by the agent.")diff --git a/letta/schemas/block.py b/letta/schemas/block.pyindex 4f7b89818..cec7c3c23 100644--- a/letta/schemas/block.py+++ b/letta/schemas/block.py@@ -110,7 +110,7 @@ class BlockResponse(Block):class FileBlock(Block):file_id: str = Field(..., description="Unique identifier of the file.")- source_id: str = Field(..., description="Unique identifier of the source.")+ source_id: str = Field(..., description="Deprecated: Use `folder_id` field instead. Unique identifier of the source.", deprecated=True)is_open: bool = Field(..., description="True if the agent currently has the file open.")last_accessed_at: Optional[datetime] = Field(None,diff --git a/letta/schemas/file.py b/letta/schemas/file.pyindex 6e305fb67..1b4102d44 100644--- a/letta/schemas/file.py+++ b/letta/schemas/file.py@@ -23,7 +23,11 @@ class FileMetadataBase(LettaBase):__id_prefix__ = PrimitiveType.FILE.value# Core file metadata fields- source_id: str = Field(..., description="The unique identifier of the source associated with the document.")+ source_id: str = Field(+ ...,+ description="Deprecated: Use `folder_id` field instead. The unique identifier of the source associated with the document.",+ deprecated=True,+ )file_name: Optional[str] = Field(None, description="The name of the file.")original_file_name: Optional[str] = Field(None, description="The original name of the file as uploaded.")file_path: Optional[str] = Field(None, description="The path to the file.")@@ -66,7 +70,7 @@ class FileAgentBase(LettaBase):# Core file-agent association fieldsagent_id: str = Field(..., description="Unique identifier of the agent.")file_id: str = Field(..., description="Unique identifier of the file.")- source_id: str = Field(..., description="Unique identifier of the source.")+ source_id: str = Field(..., description="Deprecated: Use `folder_id` field instead. Unique identifier of the source.", deprecated=True)file_name: str = Field(..., description="Name of the file.")is_open: bool = Field(True, description="True if the agent currently has the file open.")visible_content: Optional[str] = Field(diff --git a/letta/schemas/passage.py b/letta/schemas/passage.pyindex 046580c6e..39dae6b02 100644--- a/letta/schemas/passage.py+++ b/letta/schemas/passage.py@@ -21,7 +21,9 @@ class PassageBase(OrmMetadataBase):archive_id: Optional[str] = Field(None, description="The unique identifier of the archive containing this passage.")# origin data source- source_id: Optional[str] = Field(None, description="The data source of the passage.")+ source_id: Optional[str] = Field(+ None, description="Deprecated: Use `folder_id` field instead. The data source of the passage.", deprecated=True+ )# file associationfile_id: Optional[str] = Field(None, description="The unique identifier of the file associated with the passage.")diff --git a/letta/schemas/source_metadata.py b/letta/schemas/source_metadata.pyindex d395e188b..84e22c038 100644--- a/letta/schemas/source_metadata.py+++ b/letta/schemas/source_metadata.py@@ -16,8 +16,8 @@ class FileStats(LettaBase):class SourceStats(LettaBase):"""Aggregated metadata for a source"""- source_id: str = Field(..., description="Unique identifier of the source")- source_name: str = Field(..., description="Name of the source")+ source_id: str = Field(..., description="Deprecated: Use `folder_id` field instead. Unique identifier of the source", deprecated=True)+ source_name: str = Field(..., description="Deprecated: Use `folder_name` field instead. Name of the source", deprecated=True)file_count: int = Field(0, description="Number of files in the source")total_size: int = Field(0, description="Total size of all files in bytes")files: List[FileStats] = Field(default_factory=list, description="List of file statistics")diff --git a/letta/server/rest_api/routers/v1/jobs.py b/letta/server/rest_api/routers/v1/jobs.pyindex 82207a8b3..be8b57e9a 100644--- a/letta/server/rest_api/routers/v1/jobs.py+++ b/letta/server/rest_api/routers/v1/jobs.py@@ -16,7 +16,9 @@ router = APIRouter(prefix="/jobs", tags=["jobs"])@router.get("/", response_model=List[Job], operation_id="list_jobs")async def list_jobs(server: "SyncServer" = Depends(get_letta_server),- source_id: Optional[str] = Query(None, description="Only list jobs associated with the source."),+ source_id: Optional[str] = Query(+ None, description="Deprecated: Use `folder_id` parameter instead. Only list jobs associated with the source.", deprecated=True+ ),before: Optional[str] = Query(None, description="Job ID cursor for pagination. Returns jobs that come before this job ID in the specified sort order"),@@ -66,7 +68,9 @@ async def list_jobs(async def list_active_jobs(server: "SyncServer" = Depends(get_letta_server),headers: HeaderParams = Depends(get_headers),- source_id: Optional[str] = Query(None, description="Only list jobs associated with the source."),+ source_id: Optional[str] = Query(+ None, description="Deprecated: Use `folder_id` parameter instead. Only list jobs associated with the source.", deprecated=True+ ),before: Optional[str] = Query(None, description="Cursor for pagination"),after: Optional[str] = Query(None, description="Cursor for pagination"),limit: Optional[int] = Query(50, description="Limit for pagination"),diff --git a/letta/server/server.py b/letta/server/server.pyindex 6324c5644..f7577e680 100644--- a/letta/server/server.py+++ b/letta/server/server.py@@ -492,9 +492,11 @@ class SyncServer(object):log_event(name="end create_agent db")log_event(name="start insert_files_into_context_window db")- if request.source_ids:- for source_id in request.source_ids:- files = await self.file_manager.list_files(source_id, actor, include_content=True)+ # Use folder_ids if provided, otherwise fall back to deprecated source_ids for backwards compatibility+ folder_ids_to_attach = request.folder_ids if request.folder_ids else request.source_ids+ if folder_ids_to_attach:+ for folder_id in folder_ids_to_attach:+ files = await self.file_manager.list_files(folder_id, actor, include_content=True)await self.agent_manager.insert_files_into_context_window(agent_state=main_agent, file_metadata_with_content=files, actor=actor)diff --git a/letta/services/agent_manager.py b/letta/services/agent_manager.pyindex 2e41117ad..613ed77d7 100644--- a/letta/services/agent_manager.py+++ b/letta/services/agent_manager.py@@ -738,7 +738,9 @@ class AgentManager:actor: PydanticUser,) -> PydanticAgentState:new_tools = set(agent_update.tool_ids or [])- new_sources = set(agent_update.source_ids or [])+ # Use folder_ids if provided, otherwise fall back to deprecated source_ids for backwards compatibility+ folder_ids_to_update = agent_update.folder_ids if agent_update.folder_ids is not None else agent_update.source_ids+ new_sources = set(folder_ids_to_update or [])new_blocks = set(agent_update.block_ids or [])new_idents = set(agent_update.identity_ids or [])new_tags = set(agent_update.tags or [])@@ -795,7 +797,8 @@ class AgentManager:)session.expire(agent, ["tools"])- if agent_update.source_ids is not None:+ # Update sources if either folder_ids or source_ids (deprecated) is provided+ if agent_update.folder_ids is not None or agent_update.source_ids is not None:await self._replace_pivot_rows_async(session,SourcesAgents.__table__,